Add help to surveil-api command

Change-Id: I2c3b94b8d36adb2a87a63f7746be3959e320583e

doc/source/tutorials/developing_the_api.rst

@@ -30,12 +30,48 @@ Disabling permissions
 
 Depending on what you are working on, it might be practical to disable permissions. This can be done by editing the ``policy.json`` file found at ``etc/surveil/policy.json``.
 
-For example, you could modify the following line: ::
+For example, you could modify the following lines: ::
 
-        "surveil:admin": "rule:admin_required",
+    "admin_required": "role:admin or is_admin:1",
+    "surveil_required": "role:surveil or rule:admin_required",
+
+    "surveil:admin": "rule:admin_required",
+    "surveil:authenticated": "rule:surveil_required",
 
 by: ::
 
-    "surveil:admin": "rule:pass",
+    "admin_required": "@",
+    "surveil_required": "@",
+
+    "surveil:admin": "@",
+    "surveil:authenticated": "@",
 
 This will modify permissions so that all API calls that require the ``admin`` rule now pass without any verification.
+
+
+Developping the API without docker
+----------------------------------
+
+You can get development environment without docker
+
+::
+
+    git clone https://review.openstack.org/stackforge/surveil
+    cd surveil
+    virtualenv env
+    source env/bin/activate
+    pip install -r requirements.txt
+    python setup.py develop
+    python setup.py install_data
+    surveil-api -p env/etc/surveil/config.py -a env/etc/surveil/api_paste.ini -c env/etc/surveil/surveil.cfg -r
+
+Edit your config files
+
+::
+
+    vim env/etc/surveil/config.py
+    vim env/etc/surveil/surveil.cfg
+    vim env/etc/surveil/policy.json
+    vim env/etc/surveil/api_paste.ini
+
+Don't forget to start your databases (MongoDB and InfluxDB)

etc/surveil/config.py

@@ -12,7 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from six.moves import configparser
+import pecan
 
 from surveil.api import hooks
 
@@ -22,20 +22,16 @@ server = {
     'host': '0.0.0.0'
 }
 
-config = configparser.ConfigParser()
+authentication = {
-config.read("/etc/surveil/surveil.cfg")
+    'config_file': 'policy.json',
-
-surveil_api_config = {
-    "mongodb_uri": config.get("surveil", "mongodb_uri"),
-    "ws_arbiter_url": config.get("surveil", "ws_arbiter_url"),
-    "influxdb_uri": config.get("surveil", "influxdb_uri")
 }
 
+
 app_hooks = [
     hooks.DBHook(
-        surveil_api_config['mongodb_uri'],
+        pecan.conf.surveil_api_config['mongodb_uri'],
-        surveil_api_config['ws_arbiter_url'],
+        pecan.conf.surveil_api_config['ws_arbiter_url'],
-        surveil_api_config['influxdb_uri']
+        pecan.conf.surveil_api_config['influxdb_uri']
     )
 ]
 

setup.cfg

@@ -8,10 +8,12 @@ description-file =
 [files]
 packages =
     surveil
+data_files =
+    etc =  etc/*
 
 [entry_points]
 console_scripts =
-    surveil-api = surveil.cmd.api:main
+    surveil-api = surveil.cmd.api:SurveilCommandRunner.handle_command_line
     surveil-init = surveil.cmd.init:main
     surveil-pack-upload = surveil.cmd.pack_upload:main
     surveil-os-discovery = surveil.cmd.os_discovery:main

surveil/api/app.py

@@ -12,24 +12,22 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from __future__ import print_function
 from __future__ import unicode_literals
 
-import os
+import subprocess
+import sys
+import threading
+import time
 
 from paste import deploy
 import pecan
+from pecan import commands
+from pecan import configuration
+from six.moves import configparser
 
 
-def get_config_filename():
+global_pecan_config_file = None
-    abspath = os.path.abspath(__file__)
-    path = os.path.dirname(abspath)
-    filename = "config.py"
-    return os.path.join(path, filename)
-
-
-def get_pecan_config():
-    # Set up the pecan configuration
-    return pecan.configuration.conf_from_file(get_config_filename())
 
 
 def setup_app(pecan_config):
@@ -44,22 +42,116 @@ def setup_app(pecan_config):
     return app
 
 
-def load_app():
-    return deploy.loadapp('config:/etc/surveil/api_paste.ini')
-
-
 def app_factory(global_config, **local_conf):
-    return VersionSelectorApplication()
+    global global_pecan_config_file
+    return VersionSelectorApplication(global_pecan_config_file)
 
 
 class VersionSelectorApplication(object):
-    def __init__(self):
+    def __init__(self, pecan_conf_file):
-        pc = get_pecan_config()
+        pc = pecan.configuration.conf_from_file(pecan_conf_file)
 
         self.v1 = setup_app(pecan_config=pc)
         self.v2 = setup_app(pecan_config=pc)
+        self.config = pc
 
     def __call__(self, environ, start_response):
         if environ['PATH_INFO'].startswith('/v1/'):
             return self.v1(environ, start_response)
         return self.v2(environ, start_response)
+
+
+class SurveilCommand(commands.ServeCommand):
+
+    def run(self, args):
+        global global_pecan_config_file
+        global_pecan_config_file = args.pecan_config
+        super(commands.ServeCommand, self).run(args)
+        app = self.load_app()
+        self.args = args
+        self.serve(app, app.config)
+
+    def load_app(self):
+        config = configparser.ConfigParser()
+        config.read(self.args.config_file)
+        surveil_cfg = {"surveil_api_config":
+                       dict([i for i in config.items("surveil")])}
+        configuration.set_config(surveil_cfg, overwrite=True)
+        configuration.set_config(self.args.pecan_config, overwrite=False)
+
+        app = deploy.loadapp('config:%s' % self.args.api_paste_config)
+        app.config = configuration._runtime_conf
+        return app
+
+    def create_subprocess(self):
+        self.server_process = subprocess.Popen(
+            [arg for arg in sys.argv if arg not in ['--reload', '-r']],
+            stdout=sys.stdout, stderr=sys.stderr
+        )
+
+    # TODO(future useless): delete this function when
+    # https://review.openstack.org/#/c/206213/
+    # will be released
+    def watch_and_spawn(self, conf):
+        import watchdog.events as events
+        import watchdog.observers as observers
+
+        print('Monitoring for changes...',
+              file=sys.stderr)
+
+        self.create_subprocess()
+        parent = self
+
+        class AggressiveEventHandler(events.FileSystemEventHandler):
+
+            def __init__(self):
+                self.wait = False
+
+            def should_reload(self, event):
+                for t in (
+                    events.FileSystemMovedEvent,
+                    events.FileModifiedEvent,
+                    events.DirModifiedEvent
+                ):
+                    if isinstance(event, t):
+                        return True
+                return False
+
+            def ignore_events_one_sec(self):
+                if not self.wait:
+                    self.wait = True
+                    t = threading.Thread(target=self.wait_one_sec)
+                    t.start()
+
+            def wait_one_sec(self):
+                time.sleep(1)
+                self.wait = False
+
+            def on_modified(self, event):
+                if self.should_reload(event) and not self.wait:
+                    print("Some source files have been modified",
+                          file=sys.stderr)
+                    print("Restarting server...",
+                          file=sys.stderr)
+                    parent.server_process.kill()
+                    self.ignore_events_one_sec()
+                    parent.create_subprocess()
+
+        paths = self.paths_to_monitor(conf)
+
+        event_handler = AggressiveEventHandler()
+
+        for path, recurse in paths:
+            observer = observers.Observer()
+            observer.schedule(
+                event_handler,
+                path=path,
+                recursive=recurse
+            )
+            observer.start()
+
+        try:
+            while True:
+                time.sleep(1)
+        except KeyboardInterrupt:
+            pass

surveil/api/rbac.py

@@ -17,18 +17,32 @@
 
 """Access Control Lists (ACL's) control access the API server."""
 
+import os
+
 from oslo_config import cfg
 from oslo_policy import policy
+import pecan
+
+import surveil.api.app as app
 
 _ENFORCER = None
 
+policy_opts = [cfg.StrOpt('project', default='surveil')]
 
-policy_opts = [
+if app.global_pecan_config_file:
-    cfg.StrOpt('config_dir', default='/etc/surveil/'),
+    pecan_config_dir = os.path.dirname(app.global_pecan_config_file)
-    cfg.StrOpt('config_file', default='policy.json'),
+    config_dir = cfg.StrOpt('config_dir', default=pecan_config_dir)
-    cfg.StrOpt('project', default='surveil')
+else:
-]
+    config_dir = cfg.StrOpt('config_dir', default="")
+if hasattr(pecan.conf, 'authentication'):
+    config_file_name = pecan.conf.authentication.get('config_file',
+                                                     'policy.json')
+    config_file = cfg.StrOpt('config_file', default=config_file_name)
+else:
+    config_file = cfg.StrOpt('config_file', default='policy.json')
 
+policy_opts.append(config_dir)
+policy_opts.append(config_file)
 
 CONF = cfg.CONF
 

surveil/cmd/api.py

@@ -16,16 +16,14 @@
 
 from __future__ import print_function
 from __future__ import unicode_literals
+import argparse
 import os
-import subprocess
 import sys
-import threading
-import time
-from wsgiref import simple_server
 
 from oslo_config import cfg
+from pecan.commands import base
 
-import surveil.api.app as app
+from surveil.api import app
 
 CONF = cfg.CONF
 
@@ -40,100 +38,50 @@ OPTS = [
 CONF.register_opts(OPTS)
 
 
-class ServerManager:
+class SurveilCommandRunner(base.CommandRunner):
 
     def __init__(self):
-        self.config = {}
+        super(SurveilCommandRunner, self).__init__()
-        self.config_file = ""
+        self.parser = argparse.ArgumentParser(description='Surveil API server')
-        self.server_process = None
+        self.parser.add_argument('--reload', '-r', action='store_true',
-        self.should_run = True
+                                 help='Automatically reload as code changes')
+        self.parser.add_argument('--pecan_config', '-p',
+                                 default='/etc/surveil/config.py',
+                                 help='Pecan config file (config.py)')
+        self.parser.add_argument('--api_paste_config', '-a',
+                                 default='/etc/surveil/api_paste.ini',
+                                 help='API Paste config file (api_paste.ini)')
+        self.parser.add_argument('--config_file', '-c',
+                                 default='/etc/surveil/surveil.cfg',
+                                 help='Pecan config file (surveil.cfg)')
 
-    def run(self, pecan_config, config_file):
+    def run(self, args):
-        self.config = pecan_config
+        namespace = self.parser.parse_args(args)
-        self.config_file = config_file
+        # Get absolute paths
+        namespace.pecan_config = os.path.join(os.getcwd(),
+                                              namespace.pecan_config)
+        namespace.api_paste_config = os.path.join(os.getcwd(),
+                                                  namespace.api_paste_config)
+        namespace.config_file = os.path.join(os.getcwd(),
+                                             namespace.config_file)
 
-        if '--reload' in sys.argv:
+        # Check conf files exist
-            self.watch_and_spawn()
+        if not os.path.isfile(namespace.pecan_config):
-        else:
+            print("Bad config file: %s" % namespace.pecan_config,
-            self.start_server()
+                  file=sys.stderr)
+            sys.exit(1)
+        if not os.path.isfile(namespace.api_paste_config):
+            print("Bad config file: %s" % namespace.api_paste_config,
+                  file=sys.stderr)
+            sys.exit(2)
+        if not os.path.isfile(namespace.config_file):
+            print("Bad config file: %s" % namespace.config_file,
+                  file=sys.stderr)
+            sys.exit(1)
 
-    def create_subprocess(self):
+        app.SurveilCommand().run(namespace)
-        self.server_process = subprocess.Popen(['surveil-api'])
 
-    def start_server(self):
+    @classmethod
-        pecan_app = app.load_app()
+    def handle_command_line(cls):  # pragma: nocover
-        host, port = self.config.server.host, self.config.server.port
+        runner = SurveilCommandRunner()
-        srv = simple_server.make_server(host, port, pecan_app)
+        runner.run(sys.argv[1:])
-        srv.serve_forever()
-
-    def watch_and_spawn(self):
-        import watchdog.events as events
-        import watchdog.observers as observers
-
-        print('Monitoring for changes...',
-              file=sys.stderr)
-
-        self.create_subprocess()
-        parent = self
-
-        class AggressiveEventHandler(events.FileSystemEventHandler):
-
-            def __init__(self):
-                self.wait = False
-
-            def should_reload(self, event):
-                for t in (
-                    events.FileSystemMovedEvent,
-                    events.FileModifiedEvent,
-                    events.DirModifiedEvent
-                ):
-                    if isinstance(event, t):
-                        return True
-                return False
-
-            def ignore_events_one_sec(self):
-                if not self.wait:
-                    self.wait = True
-                    t = threading.Thread(target=self.wait_one_sec)
-                    t.start()
-
-            def wait_one_sec(self):
-                time.sleep(1)
-                self.wait = False
-
-            def on_modified(self, event):
-                if self.should_reload(event) and not self.wait:
-                    print("Some source files have been modified",
-                          file=sys.stderr)
-                    print("Restarting server...",
-                          file=sys.stderr)
-                    parent.server_process.kill()
-                    self.ignore_events_one_sec()
-                    parent.create_subprocess()
-
-        path = self.path_to_monitor()
-
-        event_handler = AggressiveEventHandler()
-
-        observer = observers.Observer()
-        observer.schedule(
-            event_handler,
-            path=path,
-            recursive=True
-        )
-        observer.start()
-
-        try:
-            while True:
-                time.sleep(1)
-        except KeyboardInterrupt:
-            pass
-
-    def path_to_monitor(self):
-        module = __import__('surveil')
-        return os.path.dirname(module.__file__)
-
-
-def main():
-    srv = ServerManager()
-    srv.run(app.get_pecan_config(), app.get_config_filename())

surveil/cmd/init.py

@@ -18,10 +18,12 @@ import optparse
 import sys
 
 import influxdb
+import pecan
+from pecan import configuration
 import pymongo
+from six.moves import configparser
 import surveilclient.client as sc
 
-from surveil.api import config
 from surveil.cmd import pack_upload
 
 
@@ -47,18 +49,34 @@ def main():
                       dest='packs',
                       help="Upload/Update configuration packs to MongoDB",
                       action='store_true')
+    parser.add_option('--pecan_config', '-P',
+                      default='/etc/surveil/config.py',
+                      dest='pecan_config',
+                      help='Pecan config file (config.py)')
+    parser.add_option('--config_file', '-c',
+                      default='/etc/surveil/surveil.cfg',
+                      dest='config_file',
+                      help='Pecan config file (surveil.cfg)')
+
     opts, _ = parser.parse_args(sys.argv)
 
     surveil_api_url = 'http://localhost:5311/v2'
     surveil_auth_url = 'http://localhost:5311/v2/auth'
     surveil_api_version = '2_0'
 
+    config = configparser.ConfigParser()
+    config.read(opts.config_file)
+    surveil_cfg = {"surveil_api_config":
+                   dict([i for i in config.items("surveil")])}
+    configuration.set_config(surveil_cfg, overwrite=True)
+    configuration.set_config(opts.pecan_config, overwrite=False)
+
     cli_surveil = sc.Client(surveil_api_url,
                             auth_url=surveil_auth_url,
                             version=surveil_api_version)
 
     # Create a basic config in mongodb
-    mongo = pymongo.MongoClient(config.surveil_api_config['mongodb_uri'])
+    mongo = pymongo.MongoClient(pecan.conf.surveil_api_config['mongodb_uri'])
 
     if opts.mongodb is True:
         # Drop the current shinken config
@@ -69,7 +87,7 @@ def main():
         print("Pre-creating InfluxDB database...")
         # Create the InfluxDB database
         influx_client = influxdb.InfluxDBClient.from_DSN(
-            config.surveil_api_config['influxdb_uri']
+            pecan.conf.surveil_api_config['influxdb_uri']
         )
 
         databases = influx_client.get_list_database()