From 577a1920cbb1aa672ac26da2117e934b29ac23b2 Mon Sep 17 00:00:00 2001 From: Kun Huang Date: Mon, 7 Dec 2015 13:51:11 +0800 Subject: [PATCH] enable pep8 Change-Id: Ie74349f65747cbe9703e58f071b5a12777d1e981 --- scalpels/agents/base.py | 36 +++++++++---- scalpels/agents/server.py | 85 +++++++++++++++++-------------- scalpels/client/actions/load.py | 16 ++++-- scalpels/client/actions/report.py | 18 ++++--- scalpels/client/actions/result.py | 21 ++++---- scalpels/client/actions/start.py | 12 +++-- scalpels/client/actions/stop.py | 7 ++- scalpels/client/actions/tracer.py | 12 +++-- scalpels/client/api.py | 12 +++-- scalpels/client/rpcapi.py | 6 ++- scalpels/client/shell.py | 52 ++++++++++++------- scalpels/client/utils.py | 36 +++++++------ scalpels/cmd/agent.py | 13 +++-- scalpels/cmd/manage.py | 47 +++++++++++------ scalpels/cmd/tracer.py | 24 +++++---- scalpels/db/api.py | 43 ++++++++++++---- scalpels/db/sqlalchemy/api.py | 71 ++++++++++++++++++-------- scalpels/db/sqlalchemy/models.py | 28 +++++++--- tests/unit/test_utils.py | 1 + tox.ini | 8 +-- 20 files changed, 357 insertions(+), 191 deletions(-) diff --git a/scalpels/agents/base.py b/scalpels/agents/base.py index 0fefd63..25a6a62 100644 --- a/scalpels/agents/base.py +++ b/scalpels/agents/base.py @@ -4,16 +4,20 @@ import subprocess + def run_agent(task_uuid, ag): - """ + """Example + python /agent.py mysql """ cmd = "sca-tracer %s %s" % (task_uuid, ag) ag = subprocess.Popen(cmd.split()) return ag.pid + def _parse_traffic(out, name): - """ + """Example + in: ts, 123.00 pkts 2312 bytes ... @@ -31,11 +35,11 @@ def _parse_traffic(out, name): pkts_ret = {"name": ag_name, "unit": "pkts", "data": [], - "rtype": "stream",} + "rtype": "stream"} bytes_ret = {"name": ag_name, - "unit": "bytes", - "data": [], - "rtype": "stream",} + "unit": "bytes", + "data": [], + "rtype": "stream"} for ts, _t in out: pkts, pkts_unit, bytes, bytes_unit = _t.split(" ", 3) pkts_ret["data"].append((ts, pkts)) @@ -43,16 +47,24 @@ def _parse_traffic(out, name): return (pkts_ret, bytes_ret) + def parse_rpc(out): return _parse_traffic(out, "Port") + def parse_traffic(out): return _parse_traffic(out, "Device") + def parse_rabbit(out): - """ + """Example + in: - ts, {u'_unique_id': u'xxx', u'failure': None, u'ending': True, u'result': None, u'_msg_id': u'xxx'} + ts, {u'_unique_id': u'xxx', + u'failure': None, + u'ending': True, + u'result': None, + u'_msg_id': u'xxx'} out: name: RabbitMQ unit: None @@ -64,6 +76,7 @@ def parse_rabbit(out): "data": out} return (rbt_ret, ) + def _parse_count_stream(out, name): ret = {"name": name, "unit": "count", @@ -71,8 +84,10 @@ def _parse_count_stream(out, name): "data": out} return (ret, ) + def parse_oslolock(out): - """ + """Example + in: ts, 4 ts, 0 @@ -84,11 +99,14 @@ def parse_oslolock(out): """ return _parse_count_stream(out, "Oslo-Lock") + def parse_modelsave(out): return _parse_count_stream(out, "Model-Save") + def parse_sqlaexec(out): return _parse_count_stream(out, "Sqlalchemy-Execute") + def parse_rpccount(out): return _parse_count_stream(out, "RPC-Count") diff --git a/scalpels/agents/server.py b/scalpels/agents/server.py index d29d578..77c72bc 100644 --- a/scalpels/agents/server.py +++ b/scalpels/agents/server.py @@ -1,13 +1,16 @@ -from oslo_config import cfg -import oslo_messaging -from scalpels.db import api as db_api -from scalpels.agents.base import run_agent -import psutil import signal + +from oslo_config import cfg from oslo_log import log as logging +import oslo_messaging +import psutil + +from scalpels.agents import base +from scalpels.db import api as db_api LOG = logging.getLogger(__name__) + class ServerControlEndpoint(object): target = oslo_messaging.Target(topic="test", version='1.0') @@ -19,6 +22,7 @@ class ServerControlEndpoint(object): if server: self.server.stop() + class TracerEndpoint(object): target = oslo_messaging.Target(topic="test", version='1.0') @@ -27,16 +31,17 @@ class TracerEndpoint(object): tracers = db_api.tracer_list() ret = list() for tr in tracers: - ret.append({"name":tr.name, - "tpl":tr.template, - "running":tr.is_running, - "pid":tr.pid, - "results":tr.results}) + ret.append({"name": tr.name, + "tpl": tr.template, + "running": tr.is_running, + "pid": tr.pid, + "results": tr.results}) return ret def start_tracers(self, ctx, tracers): all_tr = self.tracer_list(ctx) - running_tr = map(lambda t:t["name"], filter(lambda t:t["running"], all_tr)) + running_tr = map(lambda t: t["name"], + filter(lambda t: t["running"], all_tr)) task = db_api.task_create(results=[], pids=[]) pids = [] @@ -44,7 +49,7 @@ class TracerEndpoint(object): if tr in running_tr: LOG.info("%s is running, skipped" % tr) else: - pid = run_agent(task.uuid, tr) + pid = base.run_agent(task.uuid, tr) LOG.debug("saving pid %s" % pid) self.set_tracer_pid(ctx, tr, pid) pids.append(pid) @@ -53,7 +58,7 @@ class TracerEndpoint(object): print "[LOG] task <%s> runs successfully!" % task.uuid def stop_tracers(self, ctx, tracers): - all_tr = {t["name"]:t for t in self.tracer_list(ctx)} + all_tr = {t["name"]: t for t in self.tracer_list(ctx)} for tr in tracers: if tr not in all_tr.keys(): LOG.info("%s is not registered" % tr) @@ -66,20 +71,20 @@ class TracerEndpoint(object): p = psutil.Process(int(pid)) p.send_signal(signal.SIGINT) - - def register_tracer(self, ctx, tracer_opts): - db_api.register_tracer(name=tracer_opts["name"], template=tracer_opts["tpl"]) + db_api.register_tracer(name=tracer_opts["name"], + template=tracer_opts["tpl"]) print "[LOG] registering tracer %(name)s: %(tpl)s" % tracer_opts def set_tracer_stat(self, ctx, tracer, running): - running=bool(running) + running = bool(running) print "[LOG] setting tracer: %s running: %s" % (tracer, running) db_api.tracer_update(tracer, running=running) def set_tracer_pid(self, ctx, tracer, pid): db_api.tracer_update(tracer, pid=pid) + class TaskEndpoint(object): target = oslo_messaging.Target(topic="test", version='1.0') @@ -95,16 +100,17 @@ class TaskEndpoint(object): def get_task(self, ctx, uuid, fuzzy): print "[LOG] reading task: %s" % uuid task = db_api.task_get(uuid, fuzzy) - # TODO object - return {"uuid":task.uuid, - "results":task.results,} + # TODO(kun) object + return {"uuid": task.uuid, + "results": task.results} def get_latest_task(self, ctx): task = db_api.task_get_last() print "[LOG] reading latest task: %s" % task.uuid - # TODO object - return {"uuid":task.uuid, - "results":task.results,} + # TODO(kun) object + return {"uuid": task.uuid, + "results": task.results} + class ResultEndpoint(object): @@ -113,36 +119,37 @@ class ResultEndpoint(object): def get_result(self, ctx, uuid): print "[LOG] reading result: %s" % uuid ret = db_api.result_get(uuid) - # TODO object - return { - "id":ret.id, - "uuid":ret.uuid, - "name":ret.name, - "unit":ret.unit, - "data":ret.data, - "rtype":ret.rtype, - } + # TODO(kun) object + return {"id": ret.id, + "uuid": ret.uuid, + "name": ret.name, + "unit": ret.unit, + "data": ret.data, + "rtype": ret.rtype} def get_all_results(self, ctx): print "[LOG] reading all results" rets = db_api.get_all_results() - # TODO object - return [{"id":ret.id, - "uuid":ret.uuid, - "name":ret.name, - "unit":ret.unit, - "data":ret.data, - "rtype":ret.rtype} for ret in rets] + # TODO(kun) object + return [{"id": ret.id, + "uuid": ret.uuid, + "name": ret.name, + "unit": ret.unit, + "data": ret.data, + "rtype": ret.rtype} for ret in rets] + class ConfigEndpoint(object): target = oslo_messaging.Target(topic="test", version='1.0') + def update_config(self, ctx, data_opts): db_api.update_config(data_opts) def get_config(self, ctx): return db_api.get_config() + transport = oslo_messaging.get_transport(cfg.CONF) target = oslo_messaging.Target(topic='test', server='localhost') endpoints = [ diff --git a/scalpels/client/actions/load.py b/scalpels/client/actions/load.py index 986e4e7..4322b8d 100644 --- a/scalpels/client/actions/load.py +++ b/scalpels/client/actions/load.py @@ -2,9 +2,11 @@ #-*- coding:utf-8 -*- # Author: Kun Huang -from novaclient import client import os +from novaclient import client + + def run(config): loads = (k for k in config if config[k]) for load in loads: @@ -15,13 +17,15 @@ def run(config): continue loadcall(config) + def _get_creds_from_env(): user = os.environ.get("OS_USERNAME") pw = os.environ.get("OS_PASSWORD") tenant = os.environ.get("OS_TENANT_NAME") - auth_url= os.environ.get("OS_AUTH_URL") + auth_url = os.environ.get("OS_AUTH_URL") return (user, pw, tenant, auth_url) + def nova_boot_bulk(): creds = _get_creds_from_env() if None in creds: @@ -30,11 +34,13 @@ def nova_boot_bulk(): image = nova.images.find(name="cirros-0.3.4-x86_64-uec") flavor = nova.flavors.get("3") net = nova.networks.find(label="private") - nics = [{"net-id":net.id}] - ret = nova.servers.create(name="bulk-foo", image=image, flavor=flavor, min_count=2, max_count=2, nics=nics) + nics = [{"net-id": net.id}] + ret = nova.servers.create(name="bulk-foo", image=image, flavor=flavor, + min_count=2, max_count=2, nics=nics) print "we got %s" % ret return + def load_storm(config): - #TODO use novaclient python api to do this + #TODO(kun) use novaclient python api to do this nova_boot_bulk() diff --git a/scalpels/client/actions/report.py b/scalpels/client/actions/report.py index d6869ee..0e90e8e 100644 --- a/scalpels/client/actions/report.py +++ b/scalpels/client/actions/report.py @@ -2,10 +2,12 @@ #-*- coding:utf-8 -*- # Author: Kun Huang -from scalpels.client.api import api as agent_api -from scalpels.client.utils import generate_multiple_result_html -from scalpels.client.utils import pprint_result -from scalpels.client.actions.start import _parse_agents_from_args +from scalpels.client.actions.start import _parse_agents_from_args # noqa +from scalpels.client import api +from scalpels.client import utils + + +agent_api = api.api def run(config): @@ -13,11 +15,11 @@ def run(config): raise ValueError("Can't stop both a task and a tracer") elif config["agent"]: req_tr = _parse_agents_from_args(config) - all_tr = {t["name"]:t for t in agent_api.get_tracer_list()} + all_tr = {t["name"]: t for t in agent_api.get_tracer_list()} for tr in req_tr: for ret_uuid in all_tr[tr]["results"]: ret = agent_api.get_result(ret_uuid) - pprint_result(ret) + utils.pprint_result(ret) else: task = agent_api.try_get_task_from_config(config) @@ -28,6 +30,6 @@ def run(config): ret = agent_api.get_result(ret_uuid) rets.append(ret) if config.get("html"): - generate_multiple_result_html(rets) + utils.generate_multiple_result_html(rets) else: - map(pprint_result, rets) + map(utils.pprint_result, rets) diff --git a/scalpels/client/actions/result.py b/scalpels/client/actions/result.py index 832c59c..69201b9 100644 --- a/scalpels/client/actions/result.py +++ b/scalpels/client/actions/result.py @@ -2,13 +2,16 @@ #-*- coding:utf-8 -*- # Author: Kun Huang -from scalpels.client.utils import generate_multiple_result_html -from scalpels.client.utils import pprint_result -from scalpels.client.utils import generate_result_html -from scalpels.client.api import api as agent_api +from scalpels.client import api +from scalpels.client import utils + + +agent_api = api.api + def run(config): - """ + """Run Command + uuid: pprint it list: pprint all uuid and html: generate_result_html @@ -17,15 +20,15 @@ def run(config): if config.get("list"): rets = agent_api.get_all_results() if config.get("html"): - generate_multiple_result_html(rets) + utils.generate_multiple_result_html(rets) elif config.get("short"): for ret in rets: print ret["uuid"] else: - map(pprint_result, rets) + map(utils.pprint_result, rets) elif config.get("uuid"): ret = agent_api.get_result(config["uuid"]) if config.get("html"): - generate_result_html(ret) + utils.generate_result_html(ret) else: - pprint_result(ret) + utils.pprint_result(ret) diff --git a/scalpels/client/actions/start.py b/scalpels/client/actions/start.py index ce8fd80..056d642 100644 --- a/scalpels/client/actions/start.py +++ b/scalpels/client/actions/start.py @@ -2,9 +2,14 @@ #-*- coding:utf-8 -*- # Author: Kun Huang -import os import json -from scalpels.client.api import api as agent_api +import os + +from scalpels.client import api + + +agent_api = api.api + def _parse_agents_from_args(config): parsed_agents = set() @@ -12,7 +17,7 @@ def _parse_agents_from_args(config): return parsed_agents for ag in config.get("agent"): parsed_agents |= set(ag.split(",")) - config.update({"agent":parsed_agents}) + config.update({"agent": parsed_agents}) return parsed_agents @@ -29,6 +34,7 @@ def _parse_agents_from_file(config): parsed_agents.add(ag["name"]) return parsed_agents + def run(config): print "command start: %s" % config agents = _parse_agents_from_args(config) diff --git a/scalpels/client/actions/stop.py b/scalpels/client/actions/stop.py index ab2c355..ff1d7d6 100644 --- a/scalpels/client/actions/stop.py +++ b/scalpels/client/actions/stop.py @@ -2,8 +2,11 @@ #-*- coding:utf-8 -*- # Author: Kun Huang -from scalpels.client.api import api as agent_api -from scalpels.client.actions.start import _parse_agents_from_args +from scalpels.client.actions.start import _parse_agents_from_args # noqa +from scalpels.client import api + + +agent_api = api.api def run(config): diff --git a/scalpels/client/actions/tracer.py b/scalpels/client/actions/tracer.py index dc3d135..0c41dc6 100644 --- a/scalpels/client/actions/tracer.py +++ b/scalpels/client/actions/tracer.py @@ -2,13 +2,17 @@ #-*- coding:utf-8 -*- # Author: Kun Huang -from scalpels.client.api import api as agent_api -from prettytable import PrettyTable +import prettytable + +from scalpels.client import api + +agent_api = api.api def run(config): tracers = agent_api.get_tracer_list() - t = PrettyTable(["tracer", "tracer template", "is running", "pid"]) + t = prettytable.PrettyTable(["tracer", "tracer template", + "is running", "pid"]) for tr in tracers: - t.add_row([tr["name"],tr["tpl"],tr["running"],tr["pid"]]) + t.add_row([tr["name"], tr["tpl"], tr["running"], tr["pid"]]) print t diff --git a/scalpels/client/api.py b/scalpels/client/api.py index 038affd..806f10b 100644 --- a/scalpels/client/api.py +++ b/scalpels/client/api.py @@ -2,9 +2,12 @@ #-*- coding:utf-8 -*- # Author: Kun Huang +from scalpels.client import rpcapi +from scalpels.client import utils + +UUID_LOWEST_SUPPORT = utils.UUID_LOWEST_SUPPORT +rpcapi = rpcapi.rpcapi -from scalpels.client.rpcapi import rpcapi -from scalpels.client.utils import UUID_LOWEST_SUPPORT class API(object): def __init__(self): @@ -24,7 +27,8 @@ class API(object): def get_task(self, uuid, fuzzy=False): if fuzzy and len(uuid) < UUID_LOWEST_SUPPORT: - raise ValueError("fuzzy uuid query must get %s length" % UUID_LOWEST_SUPPORT) + raise ValueError("fuzzy uuid query must get %s length" % + UUID_LOWEST_SUPPORT) return rpcapi.get_task(uuid=uuid, fuzzy=fuzzy) def get_latest_task(self): @@ -44,7 +48,7 @@ class API(object): raise ValueError("can't assign last and uuid togther") elif uuid: return self.get_task(uuid, fuzzy=True) - else: # no matter whether last is set + else: # no matter whether last is set return self.get_latest_task() def register_tracer(self, tracer_opts): diff --git a/scalpels/client/rpcapi.py b/scalpels/client/rpcapi.py index d481461..d71afad 100644 --- a/scalpels/client/rpcapi.py +++ b/scalpels/client/rpcapi.py @@ -1,5 +1,6 @@ -import oslo_messaging as messaging from oslo_config import cfg +import oslo_messaging as messaging + class RPCAPI(object): @@ -41,7 +42,8 @@ class RPCAPI(object): return self._client.call(ctxt, "get_config") def set_tracer_stat(self, ctxt={}, tracer=None, running=None): - self._client.cast(ctxt, "set_tracer_stat", tracer=tracer, running=running) + self._client.cast(ctxt, "set_tracer_stat", tracer=tracer, + running=running) def set_tracer_pid(self, ctxt={}, tracer=None, pid=None): self._client.cast(ctxt, "set_tracer_pid", tracer=tracer, pid=pid) diff --git a/scalpels/client/shell.py b/scalpels/client/shell.py index 736abdb..eec80b5 100755 --- a/scalpels/client/shell.py +++ b/scalpels/client/shell.py @@ -13,50 +13,64 @@ def run(parser): func = getattr(mod, "run") return func(config) + def main(): - rootparser = argparse.ArgumentParser(description="main entry point for scalpels") + rootparser = argparse.ArgumentParser(description="main entry of scalpels") subparsers = rootparser.add_subparsers(title="actions", dest="action") # setup load actions load = subparsers.add_parser("load") - load.add_argument("--storm", action="store_true", dest="storm", help="run concurrency nova boot") + load.add_argument("--storm", action="store_true", dest="storm", + help="run concurrency nova boot") # setup start actions start = subparsers.add_parser("start") - start.add_argument("-f", "--file", action="store", dest="file", help="config file for this task", required=False) - start.add_argument("-a", "--agent", action="append", dest="agent", help="agent(s) to run", required=False) + start.add_argument("-f", "--file", action="store", dest="file", + help="config file for this task", required=False) + start.add_argument("-a", "--agent", action="append", dest="agent", + help="agent(s) to run", required=False) # setup stop actions stop = subparsers.add_parser("stop") - stop.add_argument("--last", action="store_true", dest="last", help="report the last task") - stop.add_argument("uuid", type=str, default="", nargs="?", help="report the last task") - stop.add_argument("-a", "--agent", action="append", dest="agent", help="agent(s) to stop", required=False) + stop.add_argument("--last", action="store_true", dest="last", + help="report the last task") + stop.add_argument("uuid", type=str, default="", nargs="?", + help="report the last task") + stop.add_argument("-a", "--agent", action="append", dest="agent", + help="agent(s) to stop", required=False) # setup report actions report = subparsers.add_parser("report") - report.add_argument("--last", action="store_true", dest="last", help="report the last task") - report.add_argument("--html", action="store_true", dest="html", help="report html to stdout instead of pretty print") - report.add_argument("uuid", type=str, default="", nargs="?", help="report the last task") - report.add_argument("-a", "--agent", action="append", dest="agent", help="agent(s) to stop", required=False) + report.add_argument("--last", action="store_true", dest="last", + help="report the last task") + report.add_argument("--html", action="store_true", dest="html", + help="report html to stdout instead of pretty print") + report.add_argument("uuid", type=str, default="", nargs="?", + help="report the last task") + report.add_argument("-a", "--agent", action="append", dest="agent", + help="agent(s) to stop", required=False) # setup sca result --list result = subparsers.add_parser("result") - result.add_argument("-l", "--list", action="store_true", dest="list", help="list all results from db") - result.add_argument("uuid", type=str, default="", nargs="?", help="report the last task") - result.add_argument("--html", action="store_true", dest="html", help="report html to stdout instead of pretty print") - result.add_argument("--short", action="store_true", dest="short", help="report uuid only") + result.add_argument("-l", "--list", action="store_true", dest="list", + help="list all results from db") + result.add_argument("uuid", type=str, default="", nargs="?", + help="report the last task") + result.add_argument("--html", action="store_true", dest="html", + help="report html to stdout instead of pretty print") + result.add_argument("--short", action="store_true", dest="short", + help="report uuid only") # agent command tracer = subparsers.add_parser("tracer") - tracer.add_argument("-l", "--list", action="store_true", dest="list", help="list all agents") - + tracer.add_argument("-l", "--list", action="store_true", dest="list", + help="list all agents") parser = rootparser.parse_args() try: run(parser) except Exception as e: - raise - return 1 + raise e else: return 0 diff --git a/scalpels/client/utils.py b/scalpels/client/utils.py index 16c098f..813fd6c 100644 --- a/scalpels/client/utils.py +++ b/scalpels/client/utils.py @@ -2,44 +2,50 @@ #-*- coding:utf-8 -*- # Author: Kun Huang -from mako.lookup import TemplateLookup -from prettytable import PrettyTable -from scalpels import templates import os +from mako import lookup as lookup_m +import prettytable + +from scalpels import templates + UUID_LOWEST_SUPPORT = 8 -# TODO this map should be saved in a config file -# TODO refar to pre/exec/post + +# TODO(kun) this map should be saved in a config file +# TODO(kun) refar to pre/exec/post tracers_map = { - "mysql": "bash %s/mysql-live.sh", #XXX doesn't work now, needs works on interapt pipeline + "mysql": "bash %s/mysql-live.sh", #XXX doesn't work now, see bug 1512276 "rabbit": "python %s/rbt-trace.py", "rpc": "bash %s/port-input-traffic.sh 5672", "traffic": "bash %s/device-input-traffic.sh eth0", - "oslolock": "stap %s/oslo-lock.stp", # with sudo, need add current user to stapdev group - "modelsave": "stap %s/model-save.stp", # with sudo, need add current user to stapdev group - "sqlaexec": "stap %s/sqla-exec.stp", # with sudo, need add current user to stapdev group - "rpccount": "stap %s/rpc-count.stp", # with sudo, need add current user to stapdev group + "oslolock": "stap %s/oslo-lock.stp", + "modelsave": "stap %s/model-save.stp", + "sqlaexec": "stap %s/sqla-exec.stp", + "rpccount": "stap %s/rpc-count.stp", } def generate_result_html(result): if result["rtype"] == "stream": - tmpl_dir = os.path.dirname(templates.__file__) - lookup = TemplateLookup(directories=[tmpl_dir]) + tmpl_dir = os.path.dirname(templates.__file__) + lookup = lookup_m.TemplateLookup(directories=[tmpl_dir]) t = lookup.get_template("line-chart.mako") print t.render(**result) + def generate_multiple_result_html(results): - tmpl_dir = os.path.dirname(templates.__file__) - lookup = TemplateLookup(directories=[tmpl_dir]) + tmpl_dir = os.path.dirname(templates.__file__) + lookup = lookup_m.TemplateLookup(directories=[tmpl_dir]) t = lookup.get_template("multi-line-chart.mako") d = {"results": results} print t.render(**d) + def pprint_result(result): print "" % result["uuid"] - t = PrettyTable(["timestamp", "%s (%s)" % (result["name"], result["unit"])]) + t = prettytable.PrettyTable(["timestamp", "%s (%s)" % (result["name"], + result["unit"])]) for data in result["data"]: t.add_row([data[0], data[1][:100]]) print t diff --git a/scalpels/cmd/agent.py b/scalpels/cmd/agent.py index 8d7a731..5d35d1c 100644 --- a/scalpels/cmd/agent.py +++ b/scalpels/cmd/agent.py @@ -2,16 +2,19 @@ #-*- coding:utf-8 -*- # Author: Kun Huang -from scalpels.agents.server import server -from oslo_log import log as logging from oslo_config import cfg +from oslo_log import log as logging + +from scalpels.agents import server + def main(): - # TODO handle stop later + # TODO(kun) handle stop later logging.register_options(cfg.CONF) logging.setup(cfg.CONF, "scalpels") - server.start() - server.wait() + server.server.start() + server.server.wait() + if __name__ == "__main__": main() diff --git a/scalpels/cmd/manage.py b/scalpels/cmd/manage.py index 7c5e8e1..8ff55ce 100644 --- a/scalpels/cmd/manage.py +++ b/scalpels/cmd/manage.py @@ -2,26 +2,33 @@ #-*- coding:utf-8 -*- # Author: Kun Huang -import os import argparse -import psutil +import os import signal + +import psutil + +from scalpels.client import api from scalpels.db import api as db_api -from scalpels.client.api import api as agent_api + + +agent_api = api.api + def get_default_tracer_dir(): # /opt/stack/data/ for devstack case # /opt/stack/ - # TODO replace these ugly codes + # TODO(kun) replace these ugly codes cmd = os.path.dirname(__file__) scalpels_py = os.path.dirname(cmd) scalpels_package = os.path.dirname(scalpels_py) default_data_dir = os.path.join(scalpels_package, "scripts") return default_data_dir + def do_db(parser): - setup_config = {"tracer_path":get_default_tracer_dir()} + setup_config = {"tracer_path": get_default_tracer_dir()} if parser.force: print "recreating database" db_api.db_drop() @@ -30,6 +37,7 @@ def do_db(parser): print "creating database" db_api.db_create(setup_config) + def do_setup(parser): data_opts = dict(parser.data_opts) if parser.data_opts else None @@ -44,14 +52,15 @@ def do_setup(parser): if parser.stat: config = agent_api.get_config() - from prettytable import PrettyTable - t = PrettyTable(["key", "value"]) - for k,v in config.items(): + import prettytable + t = prettytable.PrettyTable(["key", "value"]) + for k, v in config.items(): t.add_row([k, v]) print t + def do_stop(parser): - # TODO call rpc server's stop API instead + # TODO(kun) call rpc server's stop API instead for p in psutil.process_iter(): if p.as_dict()["cmdline"] and "sca-agent" in " ".join(p.cmdline): print "killing process %d, %s" % (p.pid, p.cmdline) @@ -60,21 +69,28 @@ def do_stop(parser): print "Can't find sca-agent process" return + def main(): - rootparser = argparse.ArgumentParser(description="main entry point for scalpels") + rootparser = argparse.ArgumentParser(description="entry point of scalpels") subparsers = rootparser.add_subparsers(title="actions", dest="action") # db actions db = subparsers.add_parser("db-create") - db.add_argument("-f", "--force", action="store_true", dest="force", help="re-create db") + db.add_argument("-f", "--force", action="store_true", + dest="force", help="re-create db") # setup re-setup actions setup = subparsers.add_parser("setup") - setup.add_argument("-d", "--data_opts", action="append", dest="data_opts", type=lambda kv:kv.split("="), help="data opts for tracer variables", required=False) - setup.add_argument("-t", "--tracer_opts", action="append", dest="tracer_opts", type=lambda kv:kv.split("="), help="tracer opts for registering", required=False) - setup.add_argument("-s", "--stat", action="store_true", dest="stat", help="setup stats for this agent") + setup.add_argument("-d", "--data_opts", action="append", dest="data_opts", + type=lambda kv: kv.split("="), + help="data opts for tracer variables", required=False) + setup.add_argument("-t", "--tracer_opts", action="append", + dest="tracer_opts", type=lambda kv: kv.split("="), + help="tracer opts for registering", required=False) + setup.add_argument("-s", "--stat", action="store_true", dest="stat", + help="setup stats for this agent") - stop = subparsers.add_parser("stop") + stop = subparsers.add_parser("stop") # noqa parser = rootparser.parse_args() @@ -87,5 +103,6 @@ def main(): if parser.action == "stop": do_stop(parser) + if __name__ == "__main__": main() diff --git a/scalpels/cmd/tracer.py b/scalpels/cmd/tracer.py index eab4837..87d5949 100644 --- a/scalpels/cmd/tracer.py +++ b/scalpels/cmd/tracer.py @@ -2,16 +2,17 @@ #-*- coding:utf-8 -*- # Author: Kun Huang -import subprocess -import psutil -import sys -from scalpels.db import api as db_api -from copy import deepcopy as copy import signal -from tooz import coordination +import subprocess +import sys import time + +import psutil +from tooz import coordination + from scalpels.agents import base -from scalpels.client.api import api as agent_api +from scalpels.client import api +from scalpels.db import api as db_api """ example: @@ -21,11 +22,13 @@ TODO: config key-word arguments for each tracer """ +agent_api = api.api worker_pid = None task_uuid = None out = None ag = None + def read_from_ag(ag): # wrong impl. here, need read from config or db instead config = agent_api.get_config() @@ -35,6 +38,7 @@ def read_from_ag(ag): return tr["tpl"] % config raise ValueError("tracer %s is not found" % ag) + def handle_int(signal, frame): stop_tracer() save_result_to_task() @@ -42,6 +46,7 @@ def handle_int(signal, frame): agent_api.set_tracer_stat(ag, running=False) sys.exit(0) + def stop_tracer(): global worker_pid # psutil is much more professional... I have to use it instead @@ -49,12 +54,13 @@ def stop_tracer(): worker_p = psutil.Process(worker_pid) worker_p.send_signal(signal.SIGINT) + def save_result_to_task(): global task_uuid global out parse_func = getattr(base, "parse_%s" % ag) - # TODO file lock is okay in localhost, here need redis for distributed + # TODO(kun) file lock is okay in localhost, here need redis for distributed # lock istead co = coordination.get_coordinator("file:///tmp", b"localhost") co.start() @@ -69,6 +75,7 @@ def save_result_to_task(): time.sleep(2) co.stop() + def main(): global worker_pid global task_uuid @@ -93,6 +100,5 @@ def main(): out.append(_t) - if __name__ == "__main__": main() diff --git a/scalpels/db/api.py b/scalpels/db/api.py index 623e118..d2ab964 100644 --- a/scalpels/db/api.py +++ b/scalpels/db/api.py @@ -3,83 +3,106 @@ # Author: Kun Huang +from oslo_config import cfg from oslo_db import api as db_api from oslo_db import options as db_options -from oslo_config import cfg + CONF = cfg.CONF +backend_mapping = {"sqlalchemy": "scalpels.db.sqlalchemy.api"} -IMPL = db_api.DBAPI("sqlalchemy", backend_mapping={"sqlalchemy":"scalpels.db.sqlalchemy.api"}) +IMPL = db_api.DBAPI("sqlalchemy", backend_mapping) + +db_options.set_defaults(CONF, + connection="sqlite:////tmp/scalpels.sqlite", + sqlite_db="scalpels.sqlite") -db_options.set_defaults(CONF, connection="sqlite:////tmp/scalpels.sqlite", sqlite_db="scalpels.sqlite") def db_create(sc): IMPL.db_create(sc) + def db_drop(): IMPL.db_drop() + def result_create(name="", unit="", data=None, rtype=None): - """ + """Create result + :param data: a list :) :returns: result model obj """ return IMPL.result_create(name, unit, data, rtype) + def task_create(results, pids): - """ + """Create task + :param results: a list contains result.uuid :returns: task model obj """ return IMPL.task_create(results, pids) + def task_get(task_uuid, fuzzy=False): return IMPL.task_get(task_uuid, fuzzy) + def task_update(task_uuid, results=None, pids=None): return IMPL.task_update(task_uuid, results, pids) + def task_append_result(task_uuid, result_uuid): return IMPL.task_append_result(task_uuid, result_uuid) + def task_get_last(): return IMPL.task_get_last() + def result_get(result_uuid): - """ + """Get result + :returns : dict, with data and its metadata """ return IMPL.result_get(result_uuid) + def setup_config_get(): - """ - :returns : dict - """ return IMPL.setup_config_get() + def get_all_results(): return IMPL.get_all_results() + def register_tracer(name, template): return IMPL.register_tracer(name, template) + def tracer_list(): return IMPL.tracer_list() + def update_config(data_opts): return IMPL.update_config(data_opts) + def get_config(): return IMPL.get_config() + def tracer_get(tracer): - """ + """Get tracer + param tracer: tracer name, like, 'rpc' """ return IMPL.tracer_get(tracer) + def tracer_update(tracer, running=None, pid=None): return IMPL.tracer_update(tracer, running, pid) + def tracer_append_result(tracer, result_uuid): return IMPL.tracer_append_result(tracer, result_uuid) diff --git a/scalpels/db/sqlalchemy/api.py b/scalpels/db/sqlalchemy/api.py index e3de7ac..fa81b03 100644 --- a/scalpels/db/sqlalchemy/api.py +++ b/scalpels/db/sqlalchemy/api.py @@ -2,18 +2,23 @@ #-*- coding:utf-8 -*- # Author: Kun Huang -from oslo_config import cfg +import copy import sys -from scalpels.db.sqlalchemy import BASE -from scalpels.db.sqlalchemy import models + +from oslo_config import cfg from oslo_db.sqlalchemy import session as db_session from oslo_db.sqlalchemy import utils as oslodbsqa_utils -from copy import deepcopy as copy + +from scalpels.db import sqlalchemy +from scalpels.db.sqlalchemy import models CONF = cfg.CONF _FACADE = None +BASE = sqlalchemy.BASE + + def _create_facade_lazily(): global _FACADE @@ -22,52 +27,61 @@ def _create_facade_lazily(): return _FACADE + def get_engine(): facade = _create_facade_lazily() return facade.get_engine() + def get_session(**kwargs): facade = _create_facade_lazily() return facade.get_session(**kwargs) + def get_backend(): return sys.modules[__name__] + def db_create(sc): BASE.metadata.create_all(get_engine()) setup = models.Setup() - setup.update({"config":sc}) + setup.update({"config": sc}) setup.save() return setup + def db_drop(): BASE.metadata.drop_all(get_engine()) + def model_query(model, session=None): if session is None: session = get_session() query = oslodbsqa_utils.model_query(model, session) return query + def result_create(name="", unit="", data=None, rtype=None): result = models.Result() - result.update({"name":name, + result.update({"name": name, "unit": unit, - "data":data, - "rtype": rtype if rtype else "unknown", - }) + "data": data, + "rtype": rtype if rtype else "unknown"}) result.save() return result + def task_create(results, pids): task = models.Task() - task.update({"results":results, "pids":pids}) + task.update({"results": results, "pids": pids}) task.save() return task + def task_update(task_uuid, results=None, pids=None): session = get_session() - task = model_query(models.Task, session=session).filter_by(uuid=task_uuid).first() + task = (model_query(models.Task, session=session). + filter_by(uuid=task_uuid).first()) _update = dict() if results: _update["results"] = results @@ -77,15 +91,18 @@ def task_update(task_uuid, results=None, pids=None): task.save(session=session) return task + def task_append_result(task_uuid, result_uuid): session = get_session() - task = model_query(models.Task, session=session).filter_by(uuid=task_uuid).first() - new = copy(task.results) + task = (model_query(models.Task, session=session). + filter_by(uuid=task_uuid).first()) + new = copy.deepcopy(task.results) new.append(result_uuid) - task.update({"results":new}) + task.update({"results": new}) task.save(session=session) return task + def task_get(task_uuid, fuzzy=False): if not fuzzy: task = model_query(models.Task).filter_by(uuid=task_uuid).first() @@ -107,12 +124,14 @@ def result_get(result_uuid): ret = model_query(models.Result).filter_by(uuid=result_uuid).first() return ret + def task_get_last(): tasks = model_query(models.Task).all() if len(tasks) > 0: return tasks[-1] return None + def setup_config_get(): setups = model_query(models.Setup).all() if len(setups) > 0: @@ -120,40 +139,48 @@ def setup_config_get(): return setup.config return None + def get_all_results(): rets = model_query(models.Result).all() return rets + def register_tracer(name, template): tracer = models.Tracer() - tracer.update({"name":name, "template": template, "results":[]}) + tracer.update({"name": name, "template": template, "results": []}) tracer.save() return tracer + def tracer_list(): tracers = model_query(models.Tracer).all() return tracers + def update_config(data_opts): session = get_session() config = model_query(models.Setup, session=session).first() - new = copy(config.config) + new = copy.deepcopy(config.config) new.update(data_opts) - config.update({"config":new}) + config.update({"config": new}) config.save(session=session) return config + def get_config(): config = model_query(models.Setup).first() return config.config + def tracer_get(tracer): tracer = model_query(models.Tracer).filter_by(name=tracer).first() return tracer + def tracer_update(tracer_name, running=None, pid=None): session = get_session() - tracer = model_query(models.Tracer, session=session).filter_by(name=tracer_name).first() + tracer = (model_query(models.Tracer, session=session). + filter_by(name=tracer_name).first()) _update = {} if running is not None: _update["is_running"] = running @@ -163,11 +190,13 @@ def tracer_update(tracer_name, running=None, pid=None): tracer.save(session=session) return tracer + def tracer_append_result(tracer_name, result_uuid): session = get_session() - tracer = model_query(models.Tracer, session=session).filter_by(name=tracer_name).first() - new = copy(tracer.results) + tracer = (model_query(models.Tracer, session=session). + filter_by(name=tracer_name).first()) + new = copy.deepcopy(tracer.results) new.append(result_uuid) - tracer.update({"results":new}) + tracer.update({"results": new}) tracer.save(session=session) return tracer diff --git a/scalpels/db/sqlalchemy/models.py b/scalpels/db/sqlalchemy/models.py index 6338558..5e695e5 100644 --- a/scalpels/db/sqlalchemy/models.py +++ b/scalpels/db/sqlalchemy/models.py @@ -2,26 +2,32 @@ #-*- coding:utf-8 -*- # Author: Kun Huang -import uuid import json +import uuid -from sqlalchemy import types as sqa_types -from scalpels.db.sqlalchemy import BASE -from sqlalchemy import (Column, Integer, String, Boolean) from oslo_db.sqlalchemy import models as oslodbmodels +from sqlalchemy import types as sqa_types +from sqlalchemy import (Column, Integer, String, Boolean) + +from scalpels.db import sqlalchemy + +BASE = sqlalchemy.BASE class JSONEncodedData(sqa_types.TypeDecorator): impl = sqa_types.Text + def process_bind_param(self, value, dialect): if value is not None: value = json.dumps(value) return value + def process_result_value(self, value, dialect): if value is not None: value = json.loads(value) return value + class ScalpelsBase(oslodbmodels.ModelBase, oslodbmodels.TimestampMixin): def save(self, session=None): if session is None: @@ -30,27 +36,33 @@ class ScalpelsBase(oslodbmodels.ModelBase, oslodbmodels.TimestampMixin): super(ScalpelsBase, self).save(session=session) + class Task(BASE, ScalpelsBase): - __tablename__ = "task" + __tablename__ = "task" id = Column(Integer, primary_key=True, autoincrement=True) - uuid = Column(String(36), default=lambda : str(uuid.uuid4()), nullable=False) + uuid = Column(String(36), default=lambda: str(uuid.uuid4()), + nullable=False) results = Column(JSONEncodedData, nullable=False) pids = Column(JSONEncodedData, nullable=False) + class Result(BASE, ScalpelsBase): - __tablename__ = "result" + __tablename__ = "result" id = Column(Integer, primary_key=True, autoincrement=True) - uuid = Column(String(36), default=lambda : str(uuid.uuid4()), nullable=False) + uuid = Column(String(36), default=lambda: str(uuid.uuid4()), + nullable=False) data = Column(JSONEncodedData, nullable=False) unit = Column(String(20), nullable=True) name = Column(String(20), nullable=True) rtype = Column(String(20), nullable=False) + class Setup(BASE, ScalpelsBase): __tablename__ = "setup" id = Column(Integer, primary_key=True, autoincrement=True) config = Column(JSONEncodedData, nullable=False) + class Tracer(BASE, ScalpelsBase): __tablename__ = "tracer" id = Column(Integer, primary_key=True, autoincrement=True) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 21810db..b2214c0 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -4,6 +4,7 @@ from oslotest import base + class TestCase(base.BaseTestCase): def setUp(self): super(TestCase, self).setUp() diff --git a/tox.ini b/tox.ini index 5189a07..dc74f2c 100644 --- a/tox.ini +++ b/tox.ini @@ -20,8 +20,8 @@ distribute = false basepython = python2.7 [testenv:pep8] -#commands = flake8 -commands = {toxinidir}/tools/fake_pep8.sh +commands = flake8 +#commands = {toxinidir}/tools/fake_pep8.sh distribute = false [testenv:py26] @@ -44,6 +44,6 @@ commands = make html downloadcache = ~/cache/pip [flake8] -ignore = H703 +ignore = H703,H102,E265,E262,H233 show-source = true -exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,build,setup.py +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,build,setup.py,tests/ci/*,scripts/*