From 39e5f67b4cd4bd02e9f40c2886151c5760515f31 Mon Sep 17 00:00:00 2001 From: Thibault Cohen Date: Mon, 27 Jul 2015 17:04:38 -0400 Subject: [PATCH] Add help to surveil-api command Change-Id: I2c3b94b8d36adb2a87a63f7746be3959e320583e --- doc/source/tutorials/developing_the_api.rst | 42 +++++- {surveil/api => etc/surveil}/config.py | 18 +-- setup.cfg | 4 +- surveil/api/app.py | 128 +++++++++++++++--- surveil/api/rbac.py | 24 +++- surveil/cmd/api.py | 140 ++++++-------------- surveil/cmd/init.py | 24 +++- 7 files changed, 243 insertions(+), 137 deletions(-) rename {surveil/api => etc/surveil}/config.py (81%) diff --git a/doc/source/tutorials/developing_the_api.rst b/doc/source/tutorials/developing_the_api.rst index 0111d9e..b5c2fc7 100644 --- a/doc/source/tutorials/developing_the_api.rst +++ b/doc/source/tutorials/developing_the_api.rst @@ -30,12 +30,48 @@ Disabling permissions Depending on what you are working on, it might be practical to disable permissions. This can be done by editing the ``policy.json`` file found at ``etc/surveil/policy.json``. -For example, you could modify the following line: :: +For example, you could modify the following lines: :: - "surveil:admin": "rule:admin_required", + "admin_required": "role:admin or is_admin:1", + "surveil_required": "role:surveil or rule:admin_required", + + "surveil:admin": "rule:admin_required", + "surveil:authenticated": "rule:surveil_required", by: :: - "surveil:admin": "rule:pass", + "admin_required": "@", + "surveil_required": "@", + + "surveil:admin": "@", + "surveil:authenticated": "@", This will modify permissions so that all API calls that require the ``admin`` rule now pass without any verification. + + +Developping the API without docker +---------------------------------- + +You can get development environment without docker + +:: + + git clone https://review.openstack.org/stackforge/surveil + cd surveil + virtualenv env + source env/bin/activate + pip install -r requirements.txt + python setup.py develop + python setup.py install_data + surveil-api -p env/etc/surveil/config.py -a env/etc/surveil/api_paste.ini -c env/etc/surveil/surveil.cfg -r + +Edit your config files + +:: + + vim env/etc/surveil/config.py + vim env/etc/surveil/surveil.cfg + vim env/etc/surveil/policy.json + vim env/etc/surveil/api_paste.ini + +Don't forget to start your databases (MongoDB and InfluxDB) diff --git a/surveil/api/config.py b/etc/surveil/config.py similarity index 81% rename from surveil/api/config.py rename to etc/surveil/config.py index f83523f..fb4cd08 100644 --- a/surveil/api/config.py +++ b/etc/surveil/config.py @@ -12,7 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. -from six.moves import configparser +import pecan from surveil.api import hooks @@ -22,20 +22,16 @@ server = { 'host': '0.0.0.0' } -config = configparser.ConfigParser() -config.read("/etc/surveil/surveil.cfg") - -surveil_api_config = { - "mongodb_uri": config.get("surveil", "mongodb_uri"), - "ws_arbiter_url": config.get("surveil", "ws_arbiter_url"), - "influxdb_uri": config.get("surveil", "influxdb_uri") +authentication = { + 'config_file': 'policy.json', } + app_hooks = [ hooks.DBHook( - surveil_api_config['mongodb_uri'], - surveil_api_config['ws_arbiter_url'], - surveil_api_config['influxdb_uri'] + pecan.conf.surveil_api_config['mongodb_uri'], + pecan.conf.surveil_api_config['ws_arbiter_url'], + pecan.conf.surveil_api_config['influxdb_uri'] ) ] diff --git a/setup.cfg b/setup.cfg index d1908fc..fb98de9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,10 +8,12 @@ description-file = [files] packages = surveil +data_files = + etc = etc/* [entry_points] console_scripts = - surveil-api = surveil.cmd.api:main + surveil-api = surveil.cmd.api:SurveilCommandRunner.handle_command_line surveil-init = surveil.cmd.init:main surveil-pack-upload = surveil.cmd.pack_upload:main surveil-os-discovery = surveil.cmd.os_discovery:main diff --git a/surveil/api/app.py b/surveil/api/app.py index e495820..f0db44d 100644 --- a/surveil/api/app.py +++ b/surveil/api/app.py @@ -12,24 +12,22 @@ # License for the specific language governing permissions and limitations # under the License. +from __future__ import print_function from __future__ import unicode_literals -import os +import subprocess +import sys +import threading +import time from paste import deploy import pecan +from pecan import commands +from pecan import configuration +from six.moves import configparser -def get_config_filename(): - abspath = os.path.abspath(__file__) - path = os.path.dirname(abspath) - filename = "config.py" - return os.path.join(path, filename) - - -def get_pecan_config(): - # Set up the pecan configuration - return pecan.configuration.conf_from_file(get_config_filename()) +global_pecan_config_file = None def setup_app(pecan_config): @@ -44,22 +42,116 @@ def setup_app(pecan_config): return app -def load_app(): - return deploy.loadapp('config:/etc/surveil/api_paste.ini') - - def app_factory(global_config, **local_conf): - return VersionSelectorApplication() + global global_pecan_config_file + return VersionSelectorApplication(global_pecan_config_file) class VersionSelectorApplication(object): - def __init__(self): - pc = get_pecan_config() + def __init__(self, pecan_conf_file): + pc = pecan.configuration.conf_from_file(pecan_conf_file) self.v1 = setup_app(pecan_config=pc) self.v2 = setup_app(pecan_config=pc) + self.config = pc def __call__(self, environ, start_response): if environ['PATH_INFO'].startswith('/v1/'): return self.v1(environ, start_response) return self.v2(environ, start_response) + + +class SurveilCommand(commands.ServeCommand): + + def run(self, args): + global global_pecan_config_file + global_pecan_config_file = args.pecan_config + super(commands.ServeCommand, self).run(args) + app = self.load_app() + self.args = args + self.serve(app, app.config) + + def load_app(self): + config = configparser.ConfigParser() + config.read(self.args.config_file) + surveil_cfg = {"surveil_api_config": + dict([i for i in config.items("surveil")])} + configuration.set_config(surveil_cfg, overwrite=True) + configuration.set_config(self.args.pecan_config, overwrite=False) + + app = deploy.loadapp('config:%s' % self.args.api_paste_config) + app.config = configuration._runtime_conf + return app + + def create_subprocess(self): + self.server_process = subprocess.Popen( + [arg for arg in sys.argv if arg not in ['--reload', '-r']], + stdout=sys.stdout, stderr=sys.stderr + ) + + # TODO(future useless): delete this function when + # https://review.openstack.org/#/c/206213/ + # will be released + def watch_and_spawn(self, conf): + import watchdog.events as events + import watchdog.observers as observers + + print('Monitoring for changes...', + file=sys.stderr) + + self.create_subprocess() + parent = self + + class AggressiveEventHandler(events.FileSystemEventHandler): + + def __init__(self): + self.wait = False + + def should_reload(self, event): + for t in ( + events.FileSystemMovedEvent, + events.FileModifiedEvent, + events.DirModifiedEvent + ): + if isinstance(event, t): + return True + return False + + def ignore_events_one_sec(self): + if not self.wait: + self.wait = True + t = threading.Thread(target=self.wait_one_sec) + t.start() + + def wait_one_sec(self): + time.sleep(1) + self.wait = False + + def on_modified(self, event): + if self.should_reload(event) and not self.wait: + print("Some source files have been modified", + file=sys.stderr) + print("Restarting server...", + file=sys.stderr) + parent.server_process.kill() + self.ignore_events_one_sec() + parent.create_subprocess() + + paths = self.paths_to_monitor(conf) + + event_handler = AggressiveEventHandler() + + for path, recurse in paths: + observer = observers.Observer() + observer.schedule( + event_handler, + path=path, + recursive=recurse + ) + observer.start() + + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + pass diff --git a/surveil/api/rbac.py b/surveil/api/rbac.py index 0b8af56..5aaba5f 100644 --- a/surveil/api/rbac.py +++ b/surveil/api/rbac.py @@ -17,18 +17,32 @@ """Access Control Lists (ACL's) control access the API server.""" +import os + from oslo_config import cfg from oslo_policy import policy +import pecan + +import surveil.api.app as app _ENFORCER = None +policy_opts = [cfg.StrOpt('project', default='surveil')] -policy_opts = [ - cfg.StrOpt('config_dir', default='/etc/surveil/'), - cfg.StrOpt('config_file', default='policy.json'), - cfg.StrOpt('project', default='surveil') -] +if app.global_pecan_config_file: + pecan_config_dir = os.path.dirname(app.global_pecan_config_file) + config_dir = cfg.StrOpt('config_dir', default=pecan_config_dir) +else: + config_dir = cfg.StrOpt('config_dir', default="") +if hasattr(pecan.conf, 'authentication'): + config_file_name = pecan.conf.authentication.get('config_file', + 'policy.json') + config_file = cfg.StrOpt('config_file', default=config_file_name) +else: + config_file = cfg.StrOpt('config_file', default='policy.json') +policy_opts.append(config_dir) +policy_opts.append(config_file) CONF = cfg.CONF diff --git a/surveil/cmd/api.py b/surveil/cmd/api.py index c384237..1092091 100644 --- a/surveil/cmd/api.py +++ b/surveil/cmd/api.py @@ -16,16 +16,14 @@ from __future__ import print_function from __future__ import unicode_literals +import argparse import os -import subprocess import sys -import threading -import time -from wsgiref import simple_server from oslo_config import cfg +from pecan.commands import base -import surveil.api.app as app +from surveil.api import app CONF = cfg.CONF @@ -40,100 +38,50 @@ OPTS = [ CONF.register_opts(OPTS) -class ServerManager: +class SurveilCommandRunner(base.CommandRunner): def __init__(self): - self.config = {} - self.config_file = "" - self.server_process = None - self.should_run = True + super(SurveilCommandRunner, self).__init__() + self.parser = argparse.ArgumentParser(description='Surveil API server') + self.parser.add_argument('--reload', '-r', action='store_true', + help='Automatically reload as code changes') + self.parser.add_argument('--pecan_config', '-p', + default='/etc/surveil/config.py', + help='Pecan config file (config.py)') + self.parser.add_argument('--api_paste_config', '-a', + default='/etc/surveil/api_paste.ini', + help='API Paste config file (api_paste.ini)') + self.parser.add_argument('--config_file', '-c', + default='/etc/surveil/surveil.cfg', + help='Pecan config file (surveil.cfg)') - def run(self, pecan_config, config_file): - self.config = pecan_config - self.config_file = config_file + def run(self, args): + namespace = self.parser.parse_args(args) + # Get absolute paths + namespace.pecan_config = os.path.join(os.getcwd(), + namespace.pecan_config) + namespace.api_paste_config = os.path.join(os.getcwd(), + namespace.api_paste_config) + namespace.config_file = os.path.join(os.getcwd(), + namespace.config_file) - if '--reload' in sys.argv: - self.watch_and_spawn() - else: - self.start_server() + # Check conf files exist + if not os.path.isfile(namespace.pecan_config): + print("Bad config file: %s" % namespace.pecan_config, + file=sys.stderr) + sys.exit(1) + if not os.path.isfile(namespace.api_paste_config): + print("Bad config file: %s" % namespace.api_paste_config, + file=sys.stderr) + sys.exit(2) + if not os.path.isfile(namespace.config_file): + print("Bad config file: %s" % namespace.config_file, + file=sys.stderr) + sys.exit(1) - def create_subprocess(self): - self.server_process = subprocess.Popen(['surveil-api']) + app.SurveilCommand().run(namespace) - def start_server(self): - pecan_app = app.load_app() - host, port = self.config.server.host, self.config.server.port - srv = simple_server.make_server(host, port, pecan_app) - srv.serve_forever() - - def watch_and_spawn(self): - import watchdog.events as events - import watchdog.observers as observers - - print('Monitoring for changes...', - file=sys.stderr) - - self.create_subprocess() - parent = self - - class AggressiveEventHandler(events.FileSystemEventHandler): - - def __init__(self): - self.wait = False - - def should_reload(self, event): - for t in ( - events.FileSystemMovedEvent, - events.FileModifiedEvent, - events.DirModifiedEvent - ): - if isinstance(event, t): - return True - return False - - def ignore_events_one_sec(self): - if not self.wait: - self.wait = True - t = threading.Thread(target=self.wait_one_sec) - t.start() - - def wait_one_sec(self): - time.sleep(1) - self.wait = False - - def on_modified(self, event): - if self.should_reload(event) and not self.wait: - print("Some source files have been modified", - file=sys.stderr) - print("Restarting server...", - file=sys.stderr) - parent.server_process.kill() - self.ignore_events_one_sec() - parent.create_subprocess() - - path = self.path_to_monitor() - - event_handler = AggressiveEventHandler() - - observer = observers.Observer() - observer.schedule( - event_handler, - path=path, - recursive=True - ) - observer.start() - - try: - while True: - time.sleep(1) - except KeyboardInterrupt: - pass - - def path_to_monitor(self): - module = __import__('surveil') - return os.path.dirname(module.__file__) - - -def main(): - srv = ServerManager() - srv.run(app.get_pecan_config(), app.get_config_filename()) + @classmethod + def handle_command_line(cls): # pragma: nocover + runner = SurveilCommandRunner() + runner.run(sys.argv[1:]) diff --git a/surveil/cmd/init.py b/surveil/cmd/init.py index 14cc34e..75fed0c 100644 --- a/surveil/cmd/init.py +++ b/surveil/cmd/init.py @@ -18,10 +18,12 @@ import optparse import sys import influxdb +import pecan +from pecan import configuration import pymongo +from six.moves import configparser import surveilclient.client as sc -from surveil.api import config from surveil.cmd import pack_upload @@ -47,18 +49,34 @@ def main(): dest='packs', help="Upload/Update configuration packs to MongoDB", action='store_true') + parser.add_option('--pecan_config', '-P', + default='/etc/surveil/config.py', + dest='pecan_config', + help='Pecan config file (config.py)') + parser.add_option('--config_file', '-c', + default='/etc/surveil/surveil.cfg', + dest='config_file', + help='Pecan config file (surveil.cfg)') + opts, _ = parser.parse_args(sys.argv) surveil_api_url = 'http://localhost:5311/v2' surveil_auth_url = 'http://localhost:5311/v2/auth' surveil_api_version = '2_0' + config = configparser.ConfigParser() + config.read(opts.config_file) + surveil_cfg = {"surveil_api_config": + dict([i for i in config.items("surveil")])} + configuration.set_config(surveil_cfg, overwrite=True) + configuration.set_config(opts.pecan_config, overwrite=False) + cli_surveil = sc.Client(surveil_api_url, auth_url=surveil_auth_url, version=surveil_api_version) # Create a basic config in mongodb - mongo = pymongo.MongoClient(config.surveil_api_config['mongodb_uri']) + mongo = pymongo.MongoClient(pecan.conf.surveil_api_config['mongodb_uri']) if opts.mongodb is True: # Drop the current shinken config @@ -69,7 +87,7 @@ def main(): print("Pre-creating InfluxDB database...") # Create the InfluxDB database influx_client = influxdb.InfluxDBClient.from_DSN( - config.surveil_api_config['influxdb_uri'] + pecan.conf.surveil_api_config['influxdb_uri'] ) databases = influx_client.get_list_database()