x/scalpels
Code Issues Proposed changes

enable pep8

Browse Source 
Change-Id: Ie74349f65747cbe9703e58f071b5a12777d1e981
This commit is contained in:
Kun Huang 2015-12-07 13:51:11 +08:00
parent 736a006fe2
commit 577a1920cb
20 changed files with 357 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
scalpels/agents/base.py
View File

@ -4,16 +4,20 @@
import subprocess
def run_agent(task_uuid, ag):
"""
"""Example
python <path-to-dir>/agent.py <uuid> mysql
"""
cmd = "sca-tracer %s %s" % (task_uuid, ag)
ag = subprocess.Popen(cmd.split())
return ag.pid
def _parse_traffic(out, name):
"""
"""Example
in:
ts, 123.00 pkts 2312 bytes
...
@ -31,11 +35,11 @@ def _parse_traffic(out, name):
pkts_ret = {"name": ag_name,
"unit": "pkts",
"data": [],
"rtype": "stream",}
"rtype": "stream"}
bytes_ret = {"name": ag_name,
"unit": "bytes",
"data": [],
"rtype": "stream",}
"rtype": "stream"}
for ts, _t in out:
pkts, pkts_unit, bytes, bytes_unit = _t.split(" ", 3)
pkts_ret["data"].append((ts, pkts))
@ -43,16 +47,24 @@ def _parse_traffic(out, name):
return (pkts_ret, bytes_ret)
def parse_rpc(out):
return _parse_traffic(out, "Port")
def parse_traffic(out):
return _parse_traffic(out, "Device")
def parse_rabbit(out):
"""
"""Example
in:
ts, {u'_unique_id': u'xxx', u'failure': None, u'ending': True, u'result': None, u'_msg_id': u'xxx'}
ts, {u'_unique_id': u'xxx',
u'failure': None,
u'ending': True,
u'result': None,
u'_msg_id': u'xxx'}
out:
name: RabbitMQ
unit: None
@ -64,6 +76,7 @@ def parse_rabbit(out):
"data": out}
return (rbt_ret, )
def _parse_count_stream(out, name):
ret = {"name": name,
"unit": "count",
@ -71,8 +84,10 @@ def _parse_count_stream(out, name):
"data": out}
return (ret, )
def parse_oslolock(out):
"""
"""Example
in:
ts, 4
ts, 0
@ -84,11 +99,14 @@ def parse_oslolock(out):
"""
return _parse_count_stream(out, "Oslo-Lock")
def parse_modelsave(out):
return _parse_count_stream(out, "Model-Save")
def parse_sqlaexec(out):
return _parse_count_stream(out, "Sqlalchemy-Execute")
def parse_rpccount(out):
return _parse_count_stream(out, "RPC-Count")

47
scalpels/agents/server.py
View File

@ -1,13 +1,16 @@
from oslo_config import cfg
import oslo_messaging
from scalpels.db import api as db_api
from scalpels.agents.base import run_agent
import psutil
import signal
from oslo_config import cfg
from oslo_log import log as logging
import oslo_messaging
import psutil
from scalpels.agents import base
from scalpels.db import api as db_api
LOG = logging.getLogger(__name__)
class ServerControlEndpoint(object):
target = oslo_messaging.Target(topic="test", version='1.0')
@ -19,6 +22,7 @@ class ServerControlEndpoint(object):
if server:
self.server.stop()
class TracerEndpoint(object):
target = oslo_messaging.Target(topic="test", version='1.0')
@ -36,7 +40,8 @@ class TracerEndpoint(object):
def start_tracers(self, ctx, tracers):
all_tr = self.tracer_list(ctx)
running_tr = map(lambda t:t["name"], filter(lambda t:t["running"], all_tr))
running_tr = map(lambda t: t["name"],
filter(lambda t: t["running"], all_tr))
task = db_api.task_create(results=[], pids=[])
pids = []
@ -44,7 +49,7 @@ class TracerEndpoint(object):
if tr in running_tr:
LOG.info("%s is running, skipped" % tr)
else:
pid = run_agent(task.uuid, tr)
pid = base.run_agent(task.uuid, tr)
LOG.debug("saving pid %s" % pid)
self.set_tracer_pid(ctx, tr, pid)
pids.append(pid)
@ -66,10 +71,9 @@ class TracerEndpoint(object):
p = psutil.Process(int(pid))
p.send_signal(signal.SIGINT)
def register_tracer(self, ctx, tracer_opts):
db_api.register_tracer(name=tracer_opts["name"], template=tracer_opts["tpl"])
db_api.register_tracer(name=tracer_opts["name"],
template=tracer_opts["tpl"])
print "[LOG] registering tracer %(name)s: %(tpl)s" % tracer_opts
def set_tracer_stat(self, ctx, tracer, running):
@ -80,6 +84,7 @@ class TracerEndpoint(object):
def set_tracer_pid(self, ctx, tracer, pid):
db_api.tracer_update(tracer, pid=pid)
class TaskEndpoint(object):
target = oslo_messaging.Target(topic="test", version='1.0')
@ -95,16 +100,17 @@ class TaskEndpoint(object):
def get_task(self, ctx, uuid, fuzzy):
print "[LOG] reading task: %s" % uuid
task = db_api.task_get(uuid, fuzzy)
# TODO object
# TODO(kun) object
return {"uuid": task.uuid,
"results":task.results,}
"results": task.results}
def get_latest_task(self, ctx):
task = db_api.task_get_last()
print "[LOG] reading latest task: %s" % task.uuid
# TODO object
# TODO(kun) object
return {"uuid": task.uuid,
"results":task.results,}
"results": task.results}
class ResultEndpoint(object):
@ -113,20 +119,18 @@ class ResultEndpoint(object):
def get_result(self, ctx, uuid):
print "[LOG] reading result: %s" % uuid
ret = db_api.result_get(uuid)
# TODO object
return {
"id":ret.id,
# TODO(kun) object
return {"id": ret.id,
"uuid": ret.uuid,
"name": ret.name,
"unit": ret.unit,
"data": ret.data,
"rtype":ret.rtype,
}
"rtype": ret.rtype}
def get_all_results(self, ctx):
print "[LOG] reading all results"
rets = db_api.get_all_results()
# TODO object
# TODO(kun) object
return [{"id": ret.id,
"uuid": ret.uuid,
"name": ret.name,
@ -134,15 +138,18 @@ class ResultEndpoint(object):
"data": ret.data,
"rtype": ret.rtype} for ret in rets]
class ConfigEndpoint(object):
target = oslo_messaging.Target(topic="test", version='1.0')
def update_config(self, ctx, data_opts):
db_api.update_config(data_opts)
def get_config(self, ctx):
return db_api.get_config()
transport = oslo_messaging.get_transport(cfg.CONF)
target = oslo_messaging.Target(topic='test', server='localhost')
endpoints = [

12
scalpels/client/actions/load.py
View File

@ -2,9 +2,11 @@
#-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com>
from novaclient import client
import os
from novaclient import client
def run(config):
loads = (k for k in config if config[k])
for load in loads:
@ -15,6 +17,7 @@ def run(config):
continue
loadcall(config)
def _get_creds_from_env():
user = os.environ.get("OS_USERNAME")
pw = os.environ.get("OS_PASSWORD")
@ -22,6 +25,7 @@ def _get_creds_from_env():
auth_url = os.environ.get("OS_AUTH_URL")
return (user, pw, tenant, auth_url)
def nova_boot_bulk():
creds = _get_creds_from_env()
if None in creds:
@ -31,10 +35,12 @@ def nova_boot_bulk():
flavor = nova.flavors.get("3")
net = nova.networks.find(label="private")
nics = [{"net-id": net.id}]
ret = nova.servers.create(name="bulk-foo", image=image, flavor=flavor, min_count=2, max_count=2, nics=nics)
ret = nova.servers.create(name="bulk-foo", image=image, flavor=flavor,
min_count=2, max_count=2, nics=nics)
print "we got %s" % ret
return
def load_storm(config):
#TODO use novaclient python api to do this
#TODO(kun) use novaclient python api to do this
nova_boot_bulk()

16
scalpels/client/actions/report.py
View File

@ -2,10 +2,12 @@
#-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com>
from scalpels.client.api import api as agent_api
from scalpels.client.utils import generate_multiple_result_html
from scalpels.client.utils import pprint_result
from scalpels.client.actions.start import _parse_agents_from_args
from scalpels.client.actions.start import _parse_agents_from_args # noqa
from scalpels.client import api
from scalpels.client import utils
agent_api = api.api
def run(config):
@ -17,7 +19,7 @@ def run(config):
for tr in req_tr:
for ret_uuid in all_tr[tr]["results"]:
ret = agent_api.get_result(ret_uuid)
pprint_result(ret)
utils.pprint_result(ret)
else:
task = agent_api.try_get_task_from_config(config)
@ -28,6 +30,6 @@ def run(config):
ret = agent_api.get_result(ret_uuid)
rets.append(ret)
if config.get("html"):
generate_multiple_result_html(rets)
utils.generate_multiple_result_html(rets)
else:
map(pprint_result, rets)
map(utils.pprint_result, rets)

21
scalpels/client/actions/result.py
View File

@ -2,13 +2,16 @@
#-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com>
from scalpels.client.utils import generate_multiple_result_html
from scalpels.client.utils import pprint_result
from scalpels.client.utils import generate_result_html
from scalpels.client.api import api as agent_api
from scalpels.client import api
from scalpels.client import utils
agent_api = api.api
def run(config):
"""
"""Run Command
uuid: pprint it
list: pprint all
uuid and html: generate_result_html
@ -17,15 +20,15 @@ def run(config):
if config.get("list"):
rets = agent_api.get_all_results()
if config.get("html"):
generate_multiple_result_html(rets)
utils.generate_multiple_result_html(rets)
elif config.get("short"):
for ret in rets:
print ret["uuid"]
else:
map(pprint_result, rets)
map(utils.pprint_result, rets)
elif config.get("uuid"):
ret = agent_api.get_result(config["uuid"])
if config.get("html"):
generate_result_html(ret)
utils.generate_result_html(ret)
else:
pprint_result(ret)
utils.pprint_result(ret)

10
scalpels/client/actions/start.py
View File

@ -2,9 +2,14 @@
#-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com>
import os
import json
from scalpels.client.api import api as agent_api
import os
from scalpels.client import api
agent_api = api.api
def _parse_agents_from_args(config):
parsed_agents = set()
@ -29,6 +34,7 @@ def _parse_agents_from_file(config):
parsed_agents.add(ag["name"])
return parsed_agents
def run(config):
print "command start: %s" % config
agents = _parse_agents_from_args(config)

7
scalpels/client/actions/stop.py
View File

@ -2,8 +2,11 @@
#-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com>
from scalpels.client.api import api as agent_api
from scalpels.client.actions.start import _parse_agents_from_args
from scalpels.client.actions.start import _parse_agents_from_args # noqa
from scalpels.client import api
agent_api = api.api
def run(config):

10
scalpels/client/actions/tracer.py
View File

@ -2,13 +2,17 @@
#-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com>
from scalpels.client.api import api as agent_api
from prettytable import PrettyTable
import prettytable
from scalpels.client import api
agent_api = api.api
def run(config):
tracers = agent_api.get_tracer_list()
t = PrettyTable(["tracer", "tracer template", "is running", "pid"])
t = prettytable.PrettyTable(["tracer", "tracer template",
"is running", "pid"])
for tr in tracers:
t.add_row([tr["name"], tr["tpl"], tr["running"], tr["pid"]])
print t

10
scalpels/client/api.py
View File

@ -2,9 +2,12 @@
#-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com>
from scalpels.client import rpcapi
from scalpels.client import utils
UUID_LOWEST_SUPPORT = utils.UUID_LOWEST_SUPPORT
rpcapi = rpcapi.rpcapi
from scalpels.client.rpcapi import rpcapi
from scalpels.client.utils import UUID_LOWEST_SUPPORT
class API(object):
def __init__(self):
@ -24,7 +27,8 @@ class API(object):
def get_task(self, uuid, fuzzy=False):
if fuzzy and len(uuid) < UUID_LOWEST_SUPPORT:
raise ValueError("fuzzy uuid query must get %s length" % UUID_LOWEST_SUPPORT)
raise ValueError("fuzzy uuid query must get %s length" %
UUID_LOWEST_SUPPORT)
return rpcapi.get_task(uuid=uuid, fuzzy=fuzzy)
def get_latest_task(self):

6
scalpels/client/rpcapi.py
View File

@ -1,5 +1,6 @@
import oslo_messaging as messaging
from oslo_config import cfg
import oslo_messaging as messaging
class RPCAPI(object):
@ -41,7 +42,8 @@ class RPCAPI(object):
return self._client.call(ctxt, "get_config")
def set_tracer_stat(self, ctxt={}, tracer=None, running=None):
self._client.cast(ctxt, "set_tracer_stat", tracer=tracer, running=running)
self._client.cast(ctxt, "set_tracer_stat", tracer=tracer,
running=running)
def set_tracer_pid(self, ctxt={}, tracer=None, pid=None):
self._client.cast(ctxt, "set_tracer_pid", tracer=tracer, pid=pid)

52
scalpels/client/shell.py
View File

@ -13,50 +13,64 @@ def run(parser):
func = getattr(mod, "run")
return func(config)
def main():
rootparser = argparse.ArgumentParser(description="main entry point for scalpels")
rootparser = argparse.ArgumentParser(description="main entry of scalpels")
subparsers = rootparser.add_subparsers(title="actions", dest="action")
# setup load actions
load = subparsers.add_parser("load")
load.add_argument("--storm", action="store_true", dest="storm", help="run concurrency nova boot")
load.add_argument("--storm", action="store_true", dest="storm",
help="run concurrency nova boot")
# setup start actions
start = subparsers.add_parser("start")
start.add_argument("-f", "--file", action="store", dest="file", help="config file for this task", required=False)
start.add_argument("-a", "--agent", action="append", dest="agent", help="agent(s) to run", required=False)
start.add_argument("-f", "--file", action="store", dest="file",
help="config file for this task", required=False)
start.add_argument("-a", "--agent", action="append", dest="agent",
help="agent(s) to run", required=False)
# setup stop actions
stop = subparsers.add_parser("stop")
stop.add_argument("--last", action="store_true", dest="last", help="report the last task")
stop.add_argument("uuid", type=str, default="", nargs="?", help="report the last task")
stop.add_argument("-a", "--agent", action="append", dest="agent", help="agent(s) to stop", required=False)
stop.add_argument("--last", action="store_true", dest="last",
help="report the last task")
stop.add_argument("uuid", type=str, default="", nargs="?",
help="report the last task")
stop.add_argument("-a", "--agent", action="append", dest="agent",
help="agent(s) to stop", required=False)
# setup report actions
report = subparsers.add_parser("report")
report.add_argument("--last", action="store_true", dest="last", help="report the last task")
report.add_argument("--html", action="store_true", dest="html", help="report html to stdout instead of pretty print")
report.add_argument("uuid", type=str, default="", nargs="?", help="report the last task")
report.add_argument("-a", "--agent", action="append", dest="agent", help="agent(s) to stop", required=False)
report.add_argument("--last", action="store_true", dest="last",
help="report the last task")
report.add_argument("--html", action="store_true", dest="html",
help="report html to stdout instead of pretty print")
report.add_argument("uuid", type=str, default="", nargs="?",
help="report the last task")
report.add_argument("-a", "--agent", action="append", dest="agent",
help="agent(s) to stop", required=False)
# setup sca result --list
result = subparsers.add_parser("result")
result.add_argument("-l", "--list", action="store_true", dest="list", help="list all results from db")
result.add_argument("uuid", type=str, default="", nargs="?", help="report the last task")
result.add_argument("--html", action="store_true", dest="html", help="report html to stdout instead of pretty print")
result.add_argument("--short", action="store_true", dest="short", help="report uuid only")
result.add_argument("-l", "--list", action="store_true", dest="list",
help="list all results from db")
result.add_argument("uuid", type=str, default="", nargs="?",
help="report the last task")
result.add_argument("--html", action="store_true", dest="html",
help="report html to stdout instead of pretty print")
result.add_argument("--short", action="store_true", dest="short",
help="report uuid only")
# agent command
tracer = subparsers.add_parser("tracer")
tracer.add_argument("-l", "--list", action="store_true", dest="list", help="list all agents")
tracer.add_argument("-l", "--list", action="store_true", dest="list",
help="list all agents")
parser = rootparser.parse_args()
try:
run(parser)
except Exception as e:
raise
return 1
raise e
else:
return 0

32
scalpels/client/utils.py
View File

@ -2,44 +2,50 @@
#-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com>
from mako.lookup import TemplateLookup
from prettytable import PrettyTable
from scalpels import templates
import os
from mako import lookup as lookup_m
import prettytable
from scalpels import templates
UUID_LOWEST_SUPPORT = 8
# TODO this map should be saved in a config file
# TODO refar to pre/exec/post
# TODO(kun) this map should be saved in a config file
# TODO(kun) refar to pre/exec/post
tracers_map = {
"mysql": "bash %s/mysql-live.sh", #XXX doesn't work now, needs works on interapt pipeline
"mysql": "bash %s/mysql-live.sh", #XXX doesn't work now, see bug 1512276
"rabbit": "python %s/rbt-trace.py",
"rpc": "bash %s/port-input-traffic.sh 5672",
"traffic": "bash %s/device-input-traffic.sh eth0",
"oslolock": "stap %s/oslo-lock.stp", # with sudo, need add current user to stapdev group
"modelsave": "stap %s/model-save.stp", # with sudo, need add current user to stapdev group
"sqlaexec": "stap %s/sqla-exec.stp", # with sudo, need add current user to stapdev group
"rpccount": "stap %s/rpc-count.stp", # with sudo, need add current user to stapdev group
"oslolock": "stap %s/oslo-lock.stp",
"modelsave": "stap %s/model-save.stp",
"sqlaexec": "stap %s/sqla-exec.stp",
"rpccount": "stap %s/rpc-count.stp",
}
def generate_result_html(result):
if result["rtype"] == "stream":
tmpl_dir = os.path.dirname(templates.__file__)
lookup = TemplateLookup(directories=[tmpl_dir])
lookup = lookup_m.TemplateLookup(directories=[tmpl_dir])
t = lookup.get_template("line-chart.mako")
print t.render(**result)
def generate_multiple_result_html(results):
tmpl_dir = os.path.dirname(templates.__file__)
lookup = TemplateLookup(directories=[tmpl_dir])
lookup = lookup_m.TemplateLookup(directories=[tmpl_dir])
t = lookup.get_template("multi-line-chart.mako")
d = {"results": results}
print t.render(**d)
def pprint_result(result):
print "<result %s>" % result["uuid"]
t = PrettyTable(["timestamp", "%s (%s)" % (result["name"], result["unit"])])
t = prettytable.PrettyTable(["timestamp", "%s (%s)" % (result["name"],
result["unit"])])
for data in result["data"]:
t.add_row([data[0], data[1][:100]])
print t

13
scalpels/cmd/agent.py
View File

@ -2,16 +2,19 @@
#-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com>
from scalpels.agents.server import server
from oslo_log import log as logging
from oslo_config import cfg
from oslo_log import log as logging
from scalpels.agents import server
def main():
# TODO handle stop later
# TODO(kun) handle stop later
logging.register_options(cfg.CONF)
logging.setup(cfg.CONF, "scalpels")
server.start()
server.wait()
server.server.start()
server.server.wait()
if __name__ == "__main__":
main()

43
scalpels/cmd/manage.py
View File

@ -2,24 +2,31 @@
#-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com>
import os
import argparse
import psutil
import os
import signal
import psutil
from scalpels.client import api
from scalpels.db import api as db_api
from scalpels.client.api import api as agent_api
agent_api = api.api
def get_default_tracer_dir():
# /opt/stack/data/ for devstack case
# /opt/stack/
# TODO replace these ugly codes
# TODO(kun) replace these ugly codes
cmd = os.path.dirname(__file__)
scalpels_py = os.path.dirname(cmd)
scalpels_package = os.path.dirname(scalpels_py)
default_data_dir = os.path.join(scalpels_package, "scripts")
return default_data_dir
def do_db(parser):
setup_config = {"tracer_path": get_default_tracer_dir()}
if parser.force:
@ -30,6 +37,7 @@ def do_db(parser):
print "creating database"
db_api.db_create(setup_config)
def do_setup(parser):
data_opts = dict(parser.data_opts) if parser.data_opts else None
@ -44,14 +52,15 @@ def do_setup(parser):
if parser.stat:
config = agent_api.get_config()
from prettytable import PrettyTable
t = PrettyTable(["key", "value"])
import prettytable
t = prettytable.PrettyTable(["key", "value"])
for k, v in config.items():
t.add_row([k, v])
print t
def do_stop(parser):
# TODO call rpc server's stop API instead
# TODO(kun) call rpc server's stop API instead
for p in psutil.process_iter():
if p.as_dict()["cmdline"] and "sca-agent" in " ".join(p.cmdline):
print "killing process %d, %s" % (p.pid, p.cmdline)
@ -60,21 +69,28 @@ def do_stop(parser):
print "Can't find sca-agent process"
return
def main():
rootparser = argparse.ArgumentParser(description="main entry point for scalpels")
rootparser = argparse.ArgumentParser(description="entry point of scalpels")
subparsers = rootparser.add_subparsers(title="actions", dest="action")
# db actions
db = subparsers.add_parser("db-create")
db.add_argument("-f", "--force", action="store_true", dest="force", help="re-create db")
db.add_argument("-f", "--force", action="store_true",
dest="force", help="re-create db")
# setup re-setup actions
setup = subparsers.add_parser("setup")
setup.add_argument("-d", "--data_opts", action="append", dest="data_opts", type=lambda kv:kv.split("="), help="data opts for tracer variables", required=False)
setup.add_argument("-t", "--tracer_opts", action="append", dest="tracer_opts", type=lambda kv:kv.split("="), help="tracer opts for registering", required=False)
setup.add_argument("-s", "--stat", action="store_true", dest="stat", help="setup stats for this agent")
setup.add_argument("-d", "--data_opts", action="append", dest="data_opts",
type=lambda kv: kv.split("="),
help="data opts for tracer variables", required=False)
setup.add_argument("-t", "--tracer_opts", action="append",
dest="tracer_opts", type=lambda kv: kv.split("="),
help="tracer opts for registering", required=False)
setup.add_argument("-s", "--stat", action="store_true", dest="stat",
help="setup stats for this agent")
stop = subparsers.add_parser("stop")
stop = subparsers.add_parser("stop") # noqa
parser = rootparser.parse_args()
@ -87,5 +103,6 @@ def main():
if parser.action == "stop":
do_stop(parser)
if __name__ == "__main__":
main()

24
scalpels/cmd/tracer.py
View File

@ -2,16 +2,17 @@
#-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com>
import subprocess
import psutil
import sys
from scalpels.db import api as db_api
from copy import deepcopy as copy
import signal
from tooz import coordination
import subprocess
import sys
import time
import psutil
from tooz import coordination
from scalpels.agents import base
from scalpels.client.api import api as agent_api
from scalpels.client import api
from scalpels.db import api as db_api
"""
example:
@ -21,11 +22,13 @@ TODO:
config key-word arguments for each tracer
"""
agent_api = api.api
worker_pid = None
task_uuid = None
out = None
ag = None
def read_from_ag(ag):
# wrong impl. here, need read from config or db instead
config = agent_api.get_config()
@ -35,6 +38,7 @@ def read_from_ag(ag):
return tr["tpl"] % config
raise ValueError("tracer %s is not found" % ag)
def handle_int(signal, frame):
stop_tracer()
save_result_to_task()
@ -42,6 +46,7 @@ def handle_int(signal, frame):
agent_api.set_tracer_stat(ag, running=False)
sys.exit(0)
def stop_tracer():
global worker_pid
# psutil is much more professional... I have to use it instead
@ -49,12 +54,13 @@ def stop_tracer():
worker_p = psutil.Process(worker_pid)
worker_p.send_signal(signal.SIGINT)
def save_result_to_task():
global task_uuid
global out
parse_func = getattr(base, "parse_%s" % ag)
# TODO file lock is okay in localhost, here need redis for distributed
# TODO(kun) file lock is okay in localhost, here need redis for distributed
# lock istead
co = coordination.get_coordinator("file:///tmp", b"localhost")
co.start()
@ -69,6 +75,7 @@ def save_result_to_task():
time.sleep(2)
co.stop()
def main():
global worker_pid
global task_uuid
@ -93,6 +100,5 @@ def main():
out.append(_t)
if __name__ == "__main__":
main()

43
scalpels/db/api.py
View File

@ -3,83 +3,106 @@
# Author: Kun Huang <academicgareth@gmail.com>
from oslo_config import cfg
from oslo_db import api as db_api
from oslo_db import options as db_options
from oslo_config import cfg
CONF = cfg.CONF
backend_mapping = {"sqlalchemy": "scalpels.db.sqlalchemy.api"}
IMPL = db_api.DBAPI("sqlalchemy", backend_mapping={"sqlalchemy":"scalpels.db.sqlalchemy.api"})
IMPL = db_api.DBAPI("sqlalchemy", backend_mapping)
db_options.set_defaults(CONF,
connection="sqlite:////tmp/scalpels.sqlite",
sqlite_db="scalpels.sqlite")
db_options.set_defaults(CONF, connection="sqlite:////tmp/scalpels.sqlite", sqlite_db="scalpels.sqlite")
def db_create(sc):
IMPL.db_create(sc)
def db_drop():
IMPL.db_drop()
def result_create(name="", unit="", data=None, rtype=None):
"""
"""Create result
:param data: a list :)
:returns: result model obj
"""
return IMPL.result_create(name, unit, data, rtype)
def task_create(results, pids):
"""
"""Create task
:param results: a list contains result.uuid
:returns: task model obj
"""
return IMPL.task_create(results, pids)
def task_get(task_uuid, fuzzy=False):
return IMPL.task_get(task_uuid, fuzzy)
def task_update(task_uuid, results=None, pids=None):
return IMPL.task_update(task_uuid, results, pids)
def task_append_result(task_uuid, result_uuid):
return IMPL.task_append_result(task_uuid, result_uuid)
def task_get_last():
return IMPL.task_get_last()
def result_get(result_uuid):
"""
"""Get result
:returns : dict, with data and its metadata
"""
return IMPL.result_get(result_uuid)
def setup_config_get():
"""
:returns : dict
"""
return IMPL.setup_config_get()
def get_all_results():
return IMPL.get_all_results()
def register_tracer(name, template):
return IMPL.register_tracer(name, template)
def tracer_list():
return IMPL.tracer_list()
def update_config(data_opts):
return IMPL.update_config(data_opts)
def get_config():
return IMPL.get_config()
def tracer_get(tracer):
"""
"""Get tracer
param tracer: tracer name, like, 'rpc'
"""
return IMPL.tracer_get(tracer)
def tracer_update(tracer, running=None, pid=None):
return IMPL.tracer_update(tracer, running, pid)
def tracer_append_result(tracer, result_uuid):
return IMPL.tracer_append_result(tracer, result_uuid)

55
scalpels/db/sqlalchemy/api.py
View File

@ -2,18 +2,23 @@
#-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com>
from oslo_config import cfg
import copy
import sys
from scalpels.db.sqlalchemy import BASE
from scalpels.db.sqlalchemy import models
from oslo_config import cfg
from oslo_db.sqlalchemy import session as db_session
from oslo_db.sqlalchemy import utils as oslodbsqa_utils
from copy import deepcopy as copy
from scalpels.db import sqlalchemy
from scalpels.db.sqlalchemy import models
CONF = cfg.CONF
_FACADE = None
BASE = sqlalchemy.BASE
def _create_facade_lazily():
global _FACADE
@ -22,17 +27,21 @@ def _create_facade_lazily():
return _FACADE
def get_engine():
facade = _create_facade_lazily()
return facade.get_engine()
def get_session(**kwargs):
facade = _create_facade_lazily()
return facade.get_session(**kwargs)
def get_backend():
return sys.modules[__name__]
def db_create(sc):
BASE.metadata.create_all(get_engine())
setup = models.Setup()
@ -40,34 +49,39 @@ def db_create(sc):
setup.save()
return setup
def db_drop():
BASE.metadata.drop_all(get_engine())
def model_query(model, session=None):
if session is None:
session = get_session()
query = oslodbsqa_utils.model_query(model, session)
return query
def result_create(name="", unit="", data=None, rtype=None):
result = models.Result()
result.update({"name": name,
"unit": unit,
"data": data,
"rtype": rtype if rtype else "unknown",
})
"rtype": rtype if rtype else "unknown"})
result.save()
return result
def task_create(results, pids):
task = models.Task()
task.update({"results": results, "pids": pids})
task.save()
return task
def task_update(task_uuid, results=None, pids=None):
session = get_session()
task = model_query(models.Task, session=session).filter_by(uuid=task_uuid).first()
task = (model_query(models.Task, session=session).
filter_by(uuid=task_uuid).first())
_update = dict()
if results:
_update["results"] = results
@ -77,15 +91,18 @@ def task_update(task_uuid, results=None, pids=None):
task.save(session=session)
return task
def task_append_result(task_uuid, result_uuid):
session = get_session()
task = model_query(models.Task, session=session).filter_by(uuid=task_uuid).first()
new = copy(task.results)
task = (model_query(models.Task, session=session).
filter_by(uuid=task_uuid).first())
new = copy.deepcopy(task.results)
new.append(result_uuid)
task.update({"results": new})
task.save(session=session)
return task
def task_get(task_uuid, fuzzy=False):
if not fuzzy:
task = model_query(models.Task).filter_by(uuid=task_uuid).first()
@ -107,12 +124,14 @@ def result_get(result_uuid):
ret = model_query(models.Result).filter_by(uuid=result_uuid).first()
return ret
def task_get_last():
tasks = model_query(models.Task).all()
if len(tasks) > 0:
return tasks[-1]
return None
def setup_config_get():
setups = model_query(models.Setup).all()
if len(setups) > 0:
@ -120,40 +139,48 @@ def setup_config_get():
return setup.config
return None
def get_all_results():
rets = model_query(models.Result).all()
return rets
def register_tracer(name, template):
tracer = models.Tracer()
tracer.update({"name": name, "template": template, "results": []})
tracer.save()
return tracer
def tracer_list():
tracers = model_query(models.Tracer).all()
return tracers
def update_config(data_opts):
session = get_session()
config = model_query(models.Setup, session=session).first()
new = copy(config.config)
new = copy.deepcopy(config.config)
new.update(data_opts)
config.update({"config": new})
config.save(session=session)
return config
def get_config():
config = model_query(models.Setup).first()
return config.config
def tracer_get(tracer):
tracer = model_query(models.Tracer).filter_by(name=tracer).first()
return tracer
def tracer_update(tracer_name, running=None, pid=None):
session = get_session()
tracer = model_query(models.Tracer, session=session).filter_by(name=tracer_name).first()
tracer = (model_query(models.Tracer, session=session).
filter_by(name=tracer_name).first())
_update = {}
if running is not None:
_update["is_running"] = running
@ -163,10 +190,12 @@ def tracer_update(tracer_name, running=None, pid=None):
tracer.save(session=session)
return tracer
def tracer_append_result(tracer_name, result_uuid):
session = get_session()
tracer = model_query(models.Tracer, session=session).filter_by(name=tracer_name).first()
new = copy(tracer.results)
tracer = (model_query(models.Tracer, session=session).
filter_by(name=tracer_name).first())
new = copy.deepcopy(tracer.results)
new.append(result_uuid)
tracer.update({"results": new})
tracer.save(session=session)

24
scalpels/db/sqlalchemy/models.py
View File

@ -2,26 +2,32 @@
#-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com>
import uuid
import json
import uuid
from sqlalchemy import types as sqa_types
from scalpels.db.sqlalchemy import BASE
from sqlalchemy import (Column, Integer, String, Boolean)
from oslo_db.sqlalchemy import models as oslodbmodels
from sqlalchemy import types as sqa_types
from sqlalchemy import (Column, Integer, String, Boolean)
from scalpels.db import sqlalchemy
BASE = sqlalchemy.BASE
class JSONEncodedData(sqa_types.TypeDecorator):
impl = sqa_types.Text
def process_bind_param(self, value, dialect):
if value is not None:
value = json.dumps(value)
return value
def process_result_value(self, value, dialect):
if value is not None:
value = json.loads(value)
return value
class ScalpelsBase(oslodbmodels.ModelBase, oslodbmodels.TimestampMixin):
def save(self, session=None):
if session is None:
@ -30,27 +36,33 @@ class ScalpelsBase(oslodbmodels.ModelBase, oslodbmodels.TimestampMixin):
super(ScalpelsBase, self).save(session=session)
class Task(BASE, ScalpelsBase):
__tablename__ = "task"
id = Column(Integer, primary_key=True, autoincrement=True)
uuid = Column(String(36), default=lambda : str(uuid.uuid4()), nullable=False)
uuid = Column(String(36), default=lambda: str(uuid.uuid4()),
nullable=False)
results = Column(JSONEncodedData, nullable=False)
pids = Column(JSONEncodedData, nullable=False)
class Result(BASE, ScalpelsBase):
__tablename__ = "result"
id = Column(Integer, primary_key=True, autoincrement=True)
uuid = Column(String(36), default=lambda : str(uuid.uuid4()), nullable=False)
uuid = Column(String(36), default=lambda: str(uuid.uuid4()),
nullable=False)
data = Column(JSONEncodedData, nullable=False)
unit = Column(String(20), nullable=True)
name = Column(String(20), nullable=True)
rtype = Column(String(20), nullable=False)
class Setup(BASE, ScalpelsBase):
__tablename__ = "setup"
id = Column(Integer, primary_key=True, autoincrement=True)
config = Column(JSONEncodedData, nullable=False)
class Tracer(BASE, ScalpelsBase):
__tablename__ = "tracer"
id = Column(Integer, primary_key=True, autoincrement=True)

1
tests/unit/test_utils.py
View File

@ -4,6 +4,7 @@
from oslotest import base
class TestCase(base.BaseTestCase):
def setUp(self):
super(TestCase, self).setUp()

8
tox.ini
View File

@ -20,8 +20,8 @@ distribute = false
basepython = python2.7
[testenv:pep8]
#commands = flake8
commands = {toxinidir}/tools/fake_pep8.sh
commands = flake8
#commands = {toxinidir}/tools/fake_pep8.sh
distribute = false
[testenv:py26]
@ -44,6 +44,6 @@ commands = make html
downloadcache = ~/cache/pip
[flake8]
ignore = H703
ignore = H703,H102,E265,E262,H233
show-source = true
exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,build,setup.py
exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,build,setup.py,tests/ci/*,scripts/*