[WIP] Add hosts APIs and compute rpc APIs

Change-Id: I5893bf066fc876e795f020214080fb92b79a00a5

devstack/README.rst

@@ -0,0 +1,9 @@
+====================
+DevStack Integration
+====================
+
+This directory contains the files necessary to integrate gyan with devstack.
+
+Refer the quickstart guide at
+https://docs.openstack.org/gyan/latest/contributor/quickstart.html
+for more information on using devstack and gyan.

devstack/lib/gyan

@@ -0,0 +1,325 @@
+#!/bin/bash
+#
+# lib/gyan
+# Functions to control the configuration and operation of the **gyan** service
+
+# Dependencies:
+#
+# - ``functions`` file
+# - ``DEST``, ``DATA_DIR``, ``STACK_USER`` must be defined
+# - ``SERVICE_{TENANT_NAME|PASSWORD}`` must be defined
+
+# ``stack.sh`` calls the entry points in this order:
+#
+# - install_gyan
+# - configure_gyan
+# - create_gyan_conf
+# - create_gyan_accounts
+# - init_gyan
+# - start_gyan
+# - stop_gyan
+# - cleanup_gyan
+
+# Save trace setting
+XTRACE=$(set +o | grep xtrace)
+set +o xtrace
+
+
+# Defaults
+# --------
+
+# Set up default directories
+GYAN_REPO=${GYAN_REPO:-${GIT_BASE}/openstack/gyan.git}
+GYAN_BRANCH=${GYAN_BRANCH:-master}
+GYAN_DIR=$DEST/gyan
+
+GITREPO["python-gyanclient"]=${GYANCLIENT_REPO:-${GIT_BASE}/openstack/python-gyanclient.git}
+GITBRANCH["python-gyanclient"]=${GYANCLIENT_BRANCH:-master}
+GITDIR["python-gyanclient"]=$DEST/python-gyanclient
+
+GYAN_STATE_PATH=${GYAN_STATE_PATH:=$DATA_DIR/gyan}
+GYAN_AUTH_CACHE_DIR=${GYAN_AUTH_CACHE_DIR:-/var/cache/gyan}
+
+GYAN_CONF_DIR=/etc/gyan
+GYAN_CONF=$GYAN_CONF_DIR/gyan.conf
+GYAN_API_PASTE=$GYAN_CONF_DIR/api-paste.ini
+
+if is_ssl_enabled_service "gyan" || is_service_enabled tls-proxy; then
+    GYAN_SERVICE_PROTOCOL="https"
+fi
+
+# Toggle for deploying GYAN-API under a wsgi server
+GYAN_USE_UWSGI=${GYAN_USE_UWSGI:-True}
+
+
+# Public facing bits
+GYAN_SERVICE_HOST=${GYAN_SERVICE_HOST:-$SERVICE_HOST}
+GYAN_SERVICE_PORT=${GYAN_SERVICE_PORT:-8517}
+GYAN_SERVICE_PORT_INT=${GYAN_SERVICE_PORT_INT:-18517}
+GYAN_SERVICE_PROTOCOL=${GYAN_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
+
+GYAN_TRUSTEE_DOMAIN_ADMIN_PASSWORD=${GYAN_TRUSTEE_DOMAIN_ADMIN_PASSWORD:-secret}
+
+# Support entry points installation of console scripts
+if [[ -d $GYAN_DIR/bin ]]; then
+    GYAN_BIN_DIR=$GYAN_DIR/bin
+else
+    GYAN_BIN_DIR=$(get_python_exec_prefix)
+fi
+
+GYAN_UWSGI=$GYAN_BIN_DIR/gyan-api-wsgi
+GYAN_UWSGI_CONF=$GYAN_CONF_DIR/gyan-api-uwsgi.ini
+
+GYAN_DB_TYPE=${GYAN_DB_TYPE:-sql}
+
+if is_ubuntu; then
+    UBUNTU_RELEASE_BASE_NUM=`lsb_release -r | awk '{print $2}' | cut -d '.' -f 1`
+fi
+
+# Functions
+# ---------
+
+# cleanup_gyan() - Remove residual data files, anything left over from previous
+# runs that a clean run would need to clean up
+function cleanup_gyan {
+    sudo rm -rf $GYAN_STATE_PATH $GYAN_AUTH_CACHE_DIR
+
+    remove_uwsgi_config "$GYAN_UWSGI_CONF" "$GYAN_UWSGI"
+}
+
+# configure_gyan() - Set config files, create data dirs, etc
+function configure_gyan {
+    # Put config files in ``/etc/gyan`` for everyone to find
+    if [[ ! -d $GYAN_CONF_DIR ]]; then
+        sudo mkdir -p $GYAN_CONF_DIR
+        sudo chown $STACK_USER $GYAN_CONF_DIR
+    fi
+
+    configure_rootwrap gyan
+
+    # Rebuild the config file from scratch
+    create_gyan_conf
+
+    create_api_paste_conf
+
+    write_uwsgi_config "$GYAN_UWSGI_CONF" "$GYAN_UWSGI" "/ml-infra"
+
+    if [[ "$USE_PYTHON3" = "True" ]]; then
+        # Switch off glance->swift communication as swift fails under py3.x
+        iniset /etc/glance/glance-api.conf glance_store default_store file
+    fi
+}
+
+
+# create_gyan_accounts() - Set up common required GYAN accounts
+#
+# Project              User         Roles
+# ------------------------------------------------------------------
+# SERVICE_PROJECT_NAME  gyan         service
+function create_gyan_accounts {
+
+    create_service_user "gyan" "admin"
+
+    if is_service_enabled gyan-api; then
+
+        local gyan_api_url
+        if [[ "$GYAN_USE_UWSGI" == "True" ]]; then
+            gyan_api_url="$GYAN_SERVICE_PROTOCOL://$GYAN_SERVICE_HOST/ml-infra"
+        else
+            gyan_api_url="$GYAN_SERVICE_PROTOCOL://$GYAN_SERVICE_HOST:$GYAN_SERVICE_PORT"
+        fi
+
+        local gyan_service=$(get_or_create_service "gyan" \
+            "ml-infra" "ML Infra As Service")
+        get_or_create_endpoint $gyan_service \
+            "$REGION_NAME" \
+            "$gyan_api_url/v1" \
+            "$gyan_api_url/v1" \
+            "$gyan_api_url/v1"
+    fi
+
+}
+
+# create_gyan_conf() - Create a new gyan.conf file
+function create_gyan_conf {
+
+    # (Re)create ``gyan.conf``
+    rm -f $GYAN_CONF
+    if [[ ${GYAN_DRIVER} == "tensorflow" ]]; then
+        iniset $GYAN_CONF DEFAULT ml_model_driver "ml_model.driver.TensorflowDriver"
+    fi
+    if [[ ${GYAN_DB_TYPE} == "sql" ]]; then
+        iniset $GYAN_CONF DEFAULT db_type sql
+    fi
+    iniset $GYAN_CONF DEFAULT debug "$ENABLE_DEBUG_LOG_LEVEL"
+    iniset $GYAN_CONF DEFAULT my_ip "$HOST_IP"
+    iniset $GYAN_CONF DEFAULT host  "$HOST_IP"
+    iniset $GYAN_CONF oslo_messaging_rabbit rabbit_userid $RABBIT_USERID
+    iniset $GYAN_CONF oslo_messaging_rabbit rabbit_password $RABBIT_PASSWORD
+    iniset $GYAN_CONF oslo_messaging_rabbit rabbit_host $RABBIT_HOST
+    iniset $GYAN_CONF database connection `database_connection_url gyan`
+    iniset $GYAN_CONF api host_ip "$GYAN_SERVICE_HOST"
+    iniset $GYAN_CONF api port "$GYAN_SERVICE_PORT"
+
+    iniset $GYAN_CONF keystone_auth auth_type password
+    iniset $GYAN_CONF keystone_auth username gyan
+    iniset $GYAN_CONF keystone_auth password $SERVICE_PASSWORD
+    iniset $GYAN_CONF keystone_auth project_name $SERVICE_PROJECT_NAME
+    iniset $GYAN_CONF keystone_auth project_domain_id default
+    iniset $GYAN_CONF keystone_auth user_domain_id default
+
+    # FIXME(pauloewerton): keystone_authtoken section is deprecated. Remove it
+    # after deprecation period.
+    iniset $GYAN_CONF keystone_authtoken admin_user gyan
+    iniset $GYAN_CONF keystone_authtoken admin_password $SERVICE_PASSWORD
+    iniset $GYAN_CONF keystone_authtoken admin_tenant_name $SERVICE_PROJECT_NAME
+
+    configure_auth_token_middleware $GYAN_CONF gyan $GYAN_AUTH_CACHE_DIR
+
+    iniset $GYAN_CONF keystone_auth auth_url $KEYSTONE_AUTH_URI_V3
+    iniset $GYAN_CONF keystone_authtoken www_authenticate_uri $KEYSTONE_SERVICE_URI_V3
+    iniset $GYAN_CONF keystone_authtoken auth_url $KEYSTONE_AUTH_URI_V3
+    iniset $GYAN_CONF keystone_authtoken auth_version v3
+
+
+    if is_fedora || is_suse; then
+        # gyan defaults to /usr/local/bin, but fedora and suse pip like to
+        # install things in /usr/bin
+        iniset $GYAN_CONF DEFAULT bindir "/usr/bin"
+    fi
+
+    if [ -n "$GYAN_STATE_PATH" ]; then
+        iniset $GYAN_CONF DEFAULT state_path "$GYAN_STATE_PATH"
+        iniset $GYAN_CONF oslo_concurrency lock_path "$GYAN_STATE_PATH"
+    fi
+
+    if [ "$SYSLOG" != "False" ]; then
+        iniset $GYAN_CONF DEFAULT use_syslog "True"
+    fi
+
+    # Format logging
+    if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then
+        setup_colorized_logging $GYAN_CONF DEFAULT
+    else
+        # Show user_name and project_name instead of user_id and project_id
+        iniset $GYAN_CONF DEFAULT logging_context_format_string "%(asctime)s.%(msecs)03d %(levelname)s %(name)s [%(request_id)s %(user_name)s %(project_name)s] %(instance)s%(message)s"
+    fi
+
+    # Register SSL certificates if provided
+    if is_ssl_enabled_service gyan; then
+        ensure_certificates gyan
+
+        iniset $GYAN_CONF DEFAULT ssl_cert_file "$GYAN_SSL_CERT"
+        iniset $GYAN_CONF DEFAULT ssl_key_file "$GYAN_SSL_KEY"
+
+        iniset $GYAN_CONF DEFAULT enabled_ssl_apis "$GYAN_ENABLED_APIS"
+    fi
+}
+
+function create_api_paste_conf {
+    # copy api_paste.ini
+    cp $GYAN_DIR/etc/gyan/api-paste.ini $GYAN_API_PASTE
+}
+
+# create_gyan_cache_dir() - Part of the init_GYAN() process
+function create_gyan_cache_dir {
+    # Create cache dir
+    sudo mkdir -p $GYAN_AUTH_CACHE_DIR
+    sudo chown $STACK_USER $GYAN_AUTH_CACHE_DIR
+    rm -f $GYAN_AUTH_CACHE_DIR/*
+}
+
+
+# init_gyan() - Initialize databases, etc.
+function init_gyan {
+    # Only do this step once on the API node for an entire cluster.
+    if is_service_enabled gyan-api; then
+        if is_service_enabled $DATABASE_BACKENDS; then
+            # (Re)create gyan database
+            recreate_database gyan
+
+            # Migrate gyan database
+            $GYAN_BIN_DIR/gyan-db-manage upgrade
+        fi
+
+        if is_service_enabled gyan-etcd; then
+            install_etcd_server
+        fi
+        create_gyan_cache_dir
+    fi
+}
+
+# install_gyanclient() - Collect source and prepare
+function install_gyanclient {
+    if use_library_from_git "python-gyanclient"; then
+        git_clone_by_name "python-gyanclient"
+        setup_dev_lib "python-gyanclient"
+        sudo install -D -m 0644 -o $STACK_USER {${GITDIR["python-gyanclient"]}/tools/,/etc/bash_completion.d/}gyan.bash_completion
+    fi
+}
+
+# install_gyan() - Collect source and prepare
+function install_gyan {
+    git_clone $GYAN_REPO $GYAN_DIR $GYAN_BRANCH
+    setup_develop $GYAN_DIR
+}
+
+# start_gyan_api() - Start the API process ahead of other things
+function start_gyan_api {
+    # Get right service port for testing
+    local service_port=$GYAN_SERVICE_PORT
+    local service_protocol=$GYAN_SERVICE_PROTOCOL
+    if is_service_enabled tls-proxy; then
+        service_port=$GYAN_SERVICE_PORT_INT
+        service_protocol="http"
+    fi
+
+    local gyan_url
+    if [ "$GYAN_USE_UWSGI" == "True" ]; then
+        run_process gyan-api "$GYAN_BIN_DIR/uwsgi --procname-prefix gyan-api --ini $GYAN_UWSGI_CONF"
+        gyan_url=$service_protocol://$GYAN_SERVICE_HOST/ml-infra
+    else
+        run_process gyan-api "$GYAN_BIN_DIR/gyan-api"
+        gyan_url=$service_protocol://$GYAN_SERVICE_HOST:$service_port
+    fi
+
+    echo "Waiting for gyan-api to start..."
+    if ! wait_for_service $SERVICE_TIMEOUT $gyan_url; then
+        die $LINENO "gyan-api did not start"
+    fi
+
+    # Start proxies if enabled
+    if is_service_enabled tls-proxy; then
+        start_tls_proxy '*' $GYAN_SERVICE_PORT $GYAN_SERVICE_HOST $GYAN_SERVICE_PORT_INT &
+    fi
+}
+
+# start_gyan_compute() - Start Gyan compute agent
+function start_gyan_compute {
+    echo "Start gyan compute..."
+    run_process gyan-compute "$GYAN_BIN_DIR/gyan-compute"
+}
+
+
+# start_gyan() - Start running processes, including screen
+function start_gyan {
+
+    # ``run_process`` checks ``is_service_enabled``, it is not needed here
+    start_gyan_api
+    start_gyan_compute
+}
+
+# stop_gyan() - Stop running processes (non-screen)
+function stop_gyan {
+
+    if [ "$GYAN_USE_UWSGI" == "True" ]; then
+        disable_apache_site gyan
+        restart_apache_server
+    else
+        stop_process gyan-api
+    fi
+    stop_process gyan-compute
+}
+
+# Restore xtrace
+$XTRACE

devstack/local.conf.sample

@@ -0,0 +1,13 @@
+[[local|localrc]]
+HOST_IP=10.0.0.11 # change this to your IP address
+DATABASE_PASSWORD=password
+RABBIT_PASSWORD=password
+SERVICE_TOKEN=password
+SERVICE_PASSWORD=password
+ADMIN_PASSWORD=password
+enable_plugin gyan https://git.openstack.org/openstack/gyan
+
+# install python-gyanclient from git
+LIBS_FROM_GIT="python-gyanclient"
+
+

devstack/local.conf.subnode.sample

@@ -0,0 +1,18 @@
+[[local|localrc]]
+HOST_IP=10.0.0.31 # change this to your IP address
+DATABASE_PASSWORD=password
+RABBIT_PASSWORD=password
+SERVICE_TOKEN=password
+SERVICE_PASSWORD=password
+ADMIN_PASSWORD=password
+enable_plugin gyan https://git.openstack.org/openstack/gyan
+
+
+# Following is for multi host settings
+MULTI_HOST=True
+SERVICE_HOST=10.0.0.11 # change this to controller's IP address
+DATABASE_TYPE=mysql
+MYSQL_HOST=$SERVICE_HOST
+RABBIT_HOST=$SERVICE_HOST
+
+ENABLED_SERVICES=gyan-compute

devstack/plugin.sh

@@ -0,0 +1,47 @@
+# gyan - Devstack extras script to install gyan
+
+# Save trace setting
+XTRACE=$(set +o | grep xtrace)
+set -o xtrace
+
+echo_summary "gyan's plugin.sh was called..."
+source $DEST/gyan/devstack/lib/gyan
+(set -o posix; set)
+
+if is_service_enabled gyan-api gyan-compute; then
+    if [[ "$1" == "stack" && "$2" == "install" ]]; then
+        echo_summary "Installing gyan"
+        install_gyan
+
+        install_gyanclient
+        cleanup_gyan
+
+    elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
+        echo_summary "Configuring gyan"
+        configure_gyan
+
+        if is_service_enabled key; then
+            create_gyan_accounts
+        fi
+
+    elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
+        # Initialize gyan
+        init_gyan
+
+        # Start the gyan API and gyan compute
+        echo_summary "Starting gyan"
+        start_gyan
+
+    fi
+
+    if [[ "$1" == "unstack" ]]; then
+        stop_gyan
+    fi
+
+    if [[ "$1" == "clean" ]]; then
+        cleanup_gyan
+    fi
+fi
+
+# Restore xtrace
+$XTRACE

devstack/settings

@@ -0,0 +1,24 @@
+# Devstack settings
+
+## Modify to your environment
+# FLOATING_RANGE=192.168.1.224/27
+# PUBLIC_NETWORK_GATEWAY=192.168.1.225
+# PUBLIC_INTERFACE=em1
+# FIXED_RANGE=10.0.0.0/24
+## Log all output to files
+# LOGFILE=$HOME/devstack.log
+## Neutron settings
+# Q_USE_SECGROUP=True
+# ENABLE_TENANT_VLANS=True
+# TENANT_VLAN_RANGE=
+# PHYSICAL_NETWORK=public
+# OVS_PHYSICAL_BRIDGE=br-ex
+
+# Enable Gyan services
+if [[ ${HOST_IP} == ${SERVICE_HOST} ]]; then
+    enable_service gyan-api
+    enable_service gyan-compute
+else
+    enable_service gyan-compute
+fi
+

etc/apache2/gyan.conf.template

@@ -0,0 +1,41 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This is an example Apache2 configuration file for using the
+# gyan API through mod_wsgi.
+
+# Note: If you are using a Debian-based system then the paths
+# "/var/log/httpd" and "/var/run/httpd" will use "apache2" instead
+# of "httpd".
+#
+# The number of processes and threads is an example only and should
+# be adjusted according to local requirements.
+
+Listen %PUBLICPORT%
+LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D(us)" gyan_combined
+
+<VirtualHost *:%PUBLICPORT%>
+    WSGIDaemonProcess gyan-api user=%USER% processes=5 threads=1 display-name=%{GROUP}
+    WSGIScriptAlias / %PUBLICWSGI%
+    WSGIProcessGroup gyan-api
+    ErrorLogFormat "%M"
+    ErrorLog /var/log/%APACHE_NAME%/gyan_api.log
+    LogLevel info
+    CustomLog /var/log/%APACHE_NAME%/gyan_access.log gyan_combined
+
+    <Directory /opt/stack/gyan/gyan/api>
+        WSGIProcessGroup gyan-api
+        WSGIApplicationGroup %{GLOBAL}
+        AllowOverride All
+        Require all granted
+    </Directory>
+</VirtualHost>

etc/gyan/api-paste.ini

@@ -0,0 +1,19 @@
+[pipeline:main]
+pipeline = cors request_id osprofiler authtoken api_v1
+
+[app:api_v1]
+paste.app_factory = gyan.api.app:app_factory
+
+[filter:authtoken]
+acl_public_routes = /, /v1
+paste.filter_factory = gyan.api.middleware.auth_token:AuthTokenMiddleware.factory
+
+[filter:osprofiler]
+paste.filter_factory = gyan.common.profiler:WsgiMiddleware.factory
+
+[filter:request_id]
+paste.filter_factory = oslo_middleware:RequestId.factory
+
+[filter:cors]
+paste.filter_factory =  oslo_middleware.cors:filter_factory
+oslo_config_project = gyan

etc/gyan/gyan-config-generator.conf

@@ -0,0 +1,14 @@
+[DEFAULT]
+output_file = etc/gyan/gyan.conf.sample
+wrap_width = 79
+
+namespace = gyan.conf
+namespace = keystonemiddleware.auth_token
+namespace = oslo.concurrency
+namespace = oslo.db
+namespace = oslo.log
+namespace = oslo.messaging
+namespace = oslo.middleware.cors
+namespace = oslo.policy
+namespace = oslo.service.periodic_task
+namespace = oslo.service.service

etc/gyan/gyan-policy-generator.conf

@@ -0,0 +1,3 @@
+[DEFAULT]
+output_file = etc/gyan/policy.yaml.sample
+namespace = gyan

etc/gyan/rootwrap.conf

@@ -0,0 +1,27 @@
+# Configuration for gyan-rootwrap
+# This file should be owned by (and only-writable by) the root user
+
+[DEFAULT]
+# List of directories to load filter definitions from (separated by ',').
+# These directories MUST all be only writable by root !
+filters_path=/etc/gyan/rootwrap.d
+
+# List of directories to search executables in, in case filters do not
+# explicitely specify a full path (separated by ',')
+# If not specified, defaults to system PATH environment variable.
+# These directories MUST all be only writable by root !
+exec_dirs=/sbin,/usr/sbin,/bin,/usr/bin,/usr/local/bin,/usr/local/sbin
+
+# Enable logging to syslog
+# Default value is False
+use_syslog=False
+
+# Which syslog facility to use.
+# Valid values include auth, authpriv, syslog, local0, local1...
+# Default value is 'syslog'
+syslog_log_facility=syslog
+
+# Which messages to log.
+# INFO means log all usage
+# ERROR means only log unsuccessful attempts
+syslog_log_level=ERROR

etc/gyan/rootwrap.d/gyan.filters

@@ -0,0 +1,8 @@
+# gyan command filters
+# This file should be owned by (and only-writeable by) the root user
+
+[Filters]
+# privileged/__init__.py: priv_context.PrivContext(default)
+# This line ties the superuser privs with the config files, context name,
+# and (implicitly) the actual python code invoked.
+privsep-rootwrap: RegExpFilter, privsep-helper, root, privsep-helper, --config-file, /etc/(?!\.\.).*, --privsep_context, os_brick.privileged.default, --privsep_sock_path, /tmp/.*

gyan/MANIFEST.in

@@ -0,0 +1 @@
+recursive-include public *

gyan/api/app.py

@@ -0,0 +1,67 @@
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import os
+
+from oslo_config import cfg
+from oslo_log import log
+from paste import deploy
+import pecan
+
+from gyan.api import config as api_config
+from gyan.api import middleware
+from gyan.common import config as common_config
+import gyan.conf
+
+CONF = gyan.conf.CONF
+LOG = log.getLogger(__name__)
+
+
+def get_pecan_config():
+    # Set up the pecan configuration
+    filename = api_config.__file__.replace('.pyc', '.py')
+    return pecan.configuration.conf_from_file(filename)
+
+
+def setup_app(config=None):
+    if not config:
+        config = get_pecan_config()
+
+    app_conf = dict(config.app)
+    common_config.set_config_defaults()
+
+    app = pecan.make_app(
+        app_conf.pop('root'),
+        logging=getattr(config, 'logging', {}),
+        wrap_app=middleware.ParsableErrorMiddleware,
+        **app_conf
+    )
+
+    return app
+
+
+def load_app():
+    cfg_file = None
+    cfg_path = CONF.api.api_paste_config
+    if not os.path.isabs(cfg_path):
+        cfg_file = CONF.find_file(cfg_path)
+    elif os.path.exists(cfg_path):
+        cfg_file = cfg_path
+
+    if not cfg_file:
+        raise cfg.ConfigFilesNotFoundError([CONF.api.api_paste_config])
+    LOG.info("Full WSGI config used: %s", cfg_file)
+    return deploy.loadapp("config:" + cfg_file)
+
+
+def app_factory(global_config, **local_conf):
+    return setup_app()

gyan/api/app.wsgi

@@ -0,0 +1,19 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Use this file for deploying the API under mod_wsgi.
+See https://pecan.readthedocs.org/en/latest/deployment.html for details.
+"""
+
+from gyan.api import wsgi
+
+application = wsgi.init_application()

gyan/api/config.py

@@ -0,0 +1,25 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from gyan.api import hooks
+
+# Pecan Application Configurations
+app = {
+    'root': 'gyan.api.controllers.root.RootController',
+    'modules': ['gyan'],
+    'hooks': [
+        hooks.ContextHook(),
+        hooks.NoExceptionTracebackHook(),
+        hooks.RPCHook(),
+    ],
+    'debug': True,
+}

gyan/api/controllers/base.py

@@ -0,0 +1,233 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import operator
+import six
+
+import pecan
+from pecan import rest
+from webob import exc
+from gyan.api.controllers import versions
+from gyan.api import versioned_method
+from gyan.common import exception
+from gyan.common.i18n import _
+
+
+# name of attribute to keep version method information
+VER_METHOD_ATTR = 'versioned_methods'
+
+
+class APIBase(object):
+
+    def __init__(self, **kwargs):
+        for field in self.fields:
+            if field in kwargs:
+                value = kwargs[field]
+                setattr(self, field, value)
+
+    def __setattr__(self, field, value):
+        super(APIBase, self).__setattr__(field, value)
+
+    def as_dict(self):
+        """Render this object as a dict of its fields."""
+        return {f: getattr(self, f)
+                for f in self.fields
+                if hasattr(self, f)}
+
+    def __json__(self):
+        return self.as_dict()
+
+    def unset_fields_except(self, except_list=None):
+        """Unset fields so they don't appear in the message body.
+
+        :param except_list: A list of fields that won't be touched.
+
+        """
+        if except_list is None:
+            except_list = []
+
+        for k in self.as_dict():
+            if k not in except_list:
+                setattr(self, k, None)
+
+
+class ControllerMetaclass(type):
+    """Controller metaclass.
+
+    This metaclass automates the task of assembling a dictionary
+    mapping action keys to method names.
+    """
+
+    def __new__(mcs, name, bases, cls_dict):
+        """Adds version function dictionary to the class."""
+
+        versioned_methods = None
+
+        for base in bases:
+            if base.__name__ == "Controller":
+                # NOTE(cyeoh): This resets the VER_METHOD_ATTR attribute
+                # between API controller class creations. This allows us
+                # to use a class decorator on the API methods that doesn't
+                # require naming explicitly what method is being versioned as
+                # it can be implicit based on the method decorated. It is a bit
+                # ugly.
+                if VER_METHOD_ATTR in base.__dict__:
+                    versioned_methods = getattr(base, VER_METHOD_ATTR)
+                    delattr(base, VER_METHOD_ATTR)
+
+        if versioned_methods:
+            cls_dict[VER_METHOD_ATTR] = versioned_methods
+
+        return super(ControllerMetaclass, mcs).__new__(mcs, name, bases,
+                                                       cls_dict)
+
+
+@six.add_metaclass(ControllerMetaclass)
+class Controller(rest.RestController):
+    """Base Rest Controller"""
+
+    @pecan.expose('json')
+    def _no_version_match(self, *args, **kwargs):
+        from pecan import request
+
+        raise exc.HTTPNotAcceptable(_(
+            "Version %(ver)s was requested but the requested API is not "
+            "supported for this version.") % {'ver': request.version})
+
+    def __getattribute__(self, key):
+
+        def version_select():
+            """Select the correct method based on version
+
+            @return: Returns the correct versioned method
+            @raises: HTTPNotAcceptable if there is no method which
+                 matches the name and version constraints
+            """
+
+            from pecan import request
+            ver = request.version
+
+            func_list = self.versioned_methods[key]
+            for func in func_list:
+                if ver.matches(func.start_version, func.end_version):
+                    return func.func
+
+            return self._no_version_match
+
+        try:
+            version_meth_dict = object.__getattribute__(self, VER_METHOD_ATTR)
+        except AttributeError:
+            # No versioning on this class
+            return object.__getattribute__(self, key)
+        if version_meth_dict and key in version_meth_dict:
+            return version_select().__get__(self, self.__class__)
+
+        return object.__getattribute__(self, key)
+
+    # NOTE: This decorator MUST appear first (the outermost
+    # decorator) on an API method for it to work correctly
+    @classmethod
+    def api_version(cls, min_ver, max_ver=None):
+        """Decorator for versioning api methods.
+
+        Add the decorator to any pecan method that has been exposed.
+        This decorator will store the method, min version, and max
+        version in a list for each api. It will check that there is no
+        overlap between versions and methods. When the api is called the
+        controller will use the list for each api to determine which
+        method to call.
+
+        Example:
+            @base.Controller.api_version("1.1", "1.2")
+            def get_one(self, ml_model_id):
+            {...code for versions 1.1 to 1.2...}
+
+            @base.Controller.api_version("1.3")
+            def get_one(self, ml_model_id):
+            {...code for versions 1.3 to latest}
+
+        @min_ver: string representing minimum version
+        @max_ver: optional string representing maximum version
+        @raises: ApiVersionsIntersect if an version overlap is found between
+        method versions.
+        """
+
+        def decorator(f):
+            obj_min_ver = versions.Version('', '', '', min_ver)
+            if max_ver:
+                obj_max_ver = versions.Version('', '', '', max_ver)
+            else:
+                obj_max_ver = versions.Version('', '', '',
+                                               versions.CURRENT_MAX_VER)
+
+            # Add to list of versioned methods registered
+            func_name = f.__name__
+            new_func = versioned_method.VersionedMethod(
+                func_name, obj_min_ver, obj_max_ver, f)
+
+            func_dict = getattr(cls, VER_METHOD_ATTR, {})
+            if not func_dict:
+                setattr(cls, VER_METHOD_ATTR, func_dict)
+
+            func_list = func_dict.get(func_name, [])
+            if not func_list:
+                func_dict[func_name] = func_list
+            func_list.append(new_func)
+
+            is_intersect = Controller.check_for_versions_intersection(
+                func_list)
+
+            if is_intersect:
+                raise exception.ApiVersionsIntersect(
+                    name=new_func.name,
+                    min_ver=new_func.start_version,
+                    max_ver=new_func.end_version
+                )
+
+            # Ensure the list is sorted by minimum version (reversed)
+            # so later when we work through the list in order we find
+            # the method which has the latest version which supports
+            # the version requested.
+            func_list.sort(key=lambda f: f.start_version, reverse=True)
+
+            return f
+
+        return decorator
+
+    @staticmethod
+    def check_for_versions_intersection(func_list):
+        """Determines whether function list intersections
+
+        General algorithm:
+        https://en.wikipedia.org/wiki/Intersection_algorithm
+
+        :param func_list: list of VersionedMethod objects
+        :return: boolean
+        """
+
+        pairs = []
+        counter = 0
+
+        for f in func_list:
+            pairs.append((f.start_version, 1))
+            pairs.append((f.end_version, -1))
+
+        pairs.sort(key=operator.itemgetter(1), reverse=True)
+        pairs.sort(key=operator.itemgetter(0))
+
+        for p in pairs:
+            counter += p[1]
+
+            if counter > 1:
+                return True
+
+        return False

gyan/api/controllers/link.py

@@ -0,0 +1,35 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import pecan
+
+
+def build_url(resource, resource_args, bookmark=False, base_url=None):
+    if base_url is None:
+        base_url = pecan.request.host_url
+
+    template = '%(url)s/%(res)s' if bookmark else '%(url)s/v1/%(res)s'
+    # FIXME(tbh): I'm getting a 404 when doing a GET on
+    # a nested resource that the URL ends with a  '/'.
+    # https://groups.google.com/forum/#!topic/pecan-dev/QfSeviLg5qs
+    template += '%(args)s' if resource_args.startswith('?') else '/%(args)s'
+    return template % {'url': base_url, 'res': resource, 'args': resource_args}
+
+
+def make_link(rel_name, url, resource, resource_args,
+              bookmark=False, type=None):
+    href = build_url(resource, resource_args,
+                     bookmark=bookmark, base_url=url)
+    if type is None:
+        return {'href': href, 'rel': rel_name}
+    else:
+        return {'href': href, 'rel': rel_name, 'type': type}

gyan/api/controllers/root.py

@@ -0,0 +1,97 @@
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import pecan
+from pecan import rest
+
+from gyan.api.controllers import base
+from gyan.api.controllers import link
+from gyan.api.controllers import v1
+from gyan.api.controllers import versions
+
+
+class Version(base.APIBase):
+    """An API version representation."""
+
+    fields = (
+        'id',
+        'links',
+        'status',
+        'max_version',
+        'min_version'
+    )
+
+    @staticmethod
+    def convert(id, status, max, min):
+        version = Version()
+        version.id = id
+        version.links = [link.make_link('self', pecan.request.host_url,
+                                        id, '', bookmark=True)]
+        version.status = status
+        version.max_version = max
+        version.min_version = min
+        return version
+
+
+class Root(base.APIBase):
+
+    fields = (
+        'name',
+        'description',
+        'versions',
+        'default_version',
+    )
+
+    @staticmethod
+    def convert():
+        root = Root()
+        root.name = "OpenStack Gyan API"
+        root.description = ("Gyan is an OpenStack project which aims to "
+                            "provide ML infra service.")
+
+        root.versions = [Version.convert('v1', "CURRENT",
+                                         versions.CURRENT_MAX_VER,
+                                         versions.BASE_VER)]
+        root.default_version = Version.convert('v1', "CURRENT",
+                                               versions.CURRENT_MAX_VER,
+                                               versions.BASE_VER)
+        return root
+
+
+class RootController(rest.RestController):
+
+    _versions = ['v1']
+    """All supported API versions"""
+
+    _default_version = 'v1'
+    """The default API version"""
+
+    v1 = v1.Controller()
+
+    @pecan.expose('json')
+    def get(self):
+        # NOTE: The reason why convert() it's being called for every
+        #       request is because we need to get the host url from
+        #       the request object to make the links.
+        return Root.convert()
+
+    @pecan.expose()
+    def _route(self, args):
+        """Overrides the default routing behavior.
+
+        It redirects the request to the default version of the gyan API
+        if the version number is not specified in the url.
+        """
+
+        if args[0] and args[0] not in self._versions:
+            args = [self._default_version] + args
+        return super(RootController, self)._route(args)

gyan/api/controllers/v1/__init__.py

@@ -0,0 +1,155 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Version 1 of the Gyan API
+
+NOTE: IN PROGRESS AND NOT FULLY IMPLEMENTED.
+"""
+
+from oslo_log import log as logging
+import pecan
+
+from gyan.api.controllers import base as controllers_base
+from gyan.api.controllers import link
+from gyan.api.controllers.v1 import hosts as host_controller
+from gyan.api.controllers.v1 import ml_models as ml_model_controller
+from gyan.api.controllers import versions as ver
+from gyan.api import http_error
+from gyan.common.i18n import _
+
+LOG = logging.getLogger(__name__)
+
+
+BASE_VERSION = 1
+
+MIN_VER_STR = '%s %s' % (ver.Version.service_string, ver.BASE_VER)
+
+MAX_VER_STR = '%s %s' % (ver.Version.service_string, ver.CURRENT_MAX_VER)
+
+MIN_VER = ver.Version({ver.Version.string: MIN_VER_STR},
+                      MIN_VER_STR, MAX_VER_STR)
+MAX_VER = ver.Version({ver.Version.string: MAX_VER_STR},
+                      MIN_VER_STR, MAX_VER_STR)
+
+
+class MediaType(controllers_base.APIBase):
+    """A media type representation."""
+
+    fields = (
+        'base',
+        'type',
+    )
+
+
+class V1(controllers_base.APIBase):
+    """The representation of the version 1 of the API."""
+
+    fields = (
+        'id',
+        'media_types',
+        'links',
+        'hosts',
+        'ml_models'
+    )
+
+    @staticmethod
+    def convert():
+        v1 = V1()
+        v1.id = "v1"
+        v1.links = [link.make_link('self', pecan.request.host_url,
+                                   'v1', '', bookmark=True),
+                    link.make_link('describedby',
+                                   'https://docs.openstack.org',
+                                   'developer/gyan/dev',
+                                   'api-spec-v1.html',
+                                   bookmark=True, type='text/html')]
+        v1.media_types = [MediaType(base='application/json',
+                          type='application/vnd.openstack.gyan.v1+json')]
+        v1.hosts = [link.make_link('self', pecan.request.host_url,
+                                        'hosts', ''),
+                         link.make_link('bookmark',
+                                        pecan.request.host_url,
+                                        'hosts', '',
+                                        bookmark=True)]
+        v1.ml_models = [link.make_link('self', pecan.request.host_url,
+                                    'ml_models', ''),
+                     link.make_link('bookmark',
+                                    pecan.request.host_url,
+                                    'ml_models', '',
+                                    bookmark=True)]
+        return v1
+
+
+class Controller(controllers_base.Controller):
+    """Version 1 API controller root."""
+
+    hosts = host_controller.HostController()
+    ml_models = ml_model_controller.MLModelController()
+
+    @pecan.expose('json')
+    def get(self):
+        return V1.convert()
+
+    def _check_version(self, version, headers=None):
+        if headers is None:
+            headers = {}
+        # ensure that major version in the URL matches the header
+        if version.major != BASE_VERSION:
+            raise http_error.HTTPNotAcceptableAPIVersion(_(
+                "Mutually exclusive versions requested. Version %(ver)s "
+                "requested but not supported by this service. "
+                "The supported version range is: "
+                "[%(min)s, %(max)s].") % {'ver': version,
+                                          'min': MIN_VER_STR,
+                                          'max': MAX_VER_STR},
+                headers=headers,
+                max_version=str(MAX_VER),
+                min_version=str(MIN_VER))
+        # ensure the minor version is within the supported range
+        if version < MIN_VER or version > MAX_VER:
+            raise http_error.HTTPNotAcceptableAPIVersion(_(
+                "Version %(ver)s was requested but the minor version is not "
+                "supported by this service. The supported version range is: "
+                "[%(min)s, %(max)s].") % {'ver': version, 'min': MIN_VER_STR,
+                                          'max': MAX_VER_STR},
+                headers=headers,
+                max_version=str(MAX_VER),
+                min_version=str(MIN_VER))
+
+    @pecan.expose()
+    def _route(self, args):
+        version = ver.Version(
+            pecan.request.headers, MIN_VER_STR, MAX_VER_STR)
+
+        # Always set the basic version headers
+        pecan.response.headers[ver.Version.min_string] = MIN_VER_STR
+        pecan.response.headers[ver.Version.max_string] = MAX_VER_STR
+        pecan.response.headers[ver.Version.string] = " ".join(
+            [ver.Version.service_string, str(version)])
+        pecan.response.headers["vary"] = ver.Version.string
+
+        # assert that requested version is supported
+        self._check_version(version, pecan.response.headers)
+        pecan.request.version = version
+        if pecan.request.body:
+            msg = ("Processing request: url: %(url)s, %(method)s, "
+                   "body: %(body)s" %
+                   {'url': pecan.request.url,
+                    'method': pecan.request.method,
+                    'body': pecan.request.body})
+            LOG.debug(msg)
+
+        return super(Controller, self)._route(args)
+
+
+__all__ = ('Controller',)

gyan/api/controllers/v1/collection.py

@@ -0,0 +1,41 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import pecan
+
+from gyan.api.controllers import base
+from gyan.api.controllers import link
+
+
+class Collection(base.APIBase):
+
+    @property
+    def collection(self):
+        return getattr(self, self._type)
+
+    def has_next(self, limit):
+        """Return whether collection has more items."""
+        return len(self.collection) and len(self.collection) == limit
+
+    def get_next(self, limit, url=None, **kwargs):
+        """Return a link to the next subset of the collection."""
+        if not self.has_next(limit):
+            return None
+
+        resource_url = url or self._type
+        q_args = ''.join(['%s=%s&' % (key, kwargs[key]) for key in kwargs])
+        next_args = '?%(args)slimit=%(limit)d&marker=%(marker)s' % {
+            'args': q_args, 'limit': limit,
+            'marker': self.collection[-1]['uuid']}
+
+        return link.make_link('next', pecan.request.host_url,
+                              resource_url, next_args)['href']

gyan/api/controllers/v1/hosts.py

@@ -0,0 +1,109 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import pecan
+
+from gyan.api.controllers import base
+from gyan.api.controllers.v1 import collection
+from gyan.api.controllers.v1.views import hosts_view as view
+from gyan.api import utils as api_utils
+from gyan.common import exception
+from gyan.common import policy
+from gyan import objects
+
+
+def _get_host(host_ident):
+    host = api_utils.get_resource('ComputeHost', host_ident)
+    if not host:
+        pecan.abort(404, ('Not found; the host you requested '
+                          'does not exist.'))
+
+    return host
+
+
+def check_policy_on_host(host, action):
+    context = pecan.request.context
+    policy.enforce(context, action, host, action=action)
+
+
+class HostCollection(collection.Collection):
+    """API representation of a collection of hosts."""
+
+    fields = {
+        'hosts',
+        'next'
+    }
+
+    """A list containing compute host objects"""
+
+    def __init__(self, **kwargs):
+        super(HostCollection, self).__init__(**kwargs)
+        self._type = 'hosts'
+
+    @staticmethod
+    def convert_with_links(hosts, limit, url=None,
+                           expand=False, **kwargs):
+        collection = HostCollection()
+        collection.hosts = [view.format_host(url, p) for p in hosts]
+        collection.next = collection.get_next(limit, url=url, **kwargs)
+        return collection
+
+
+class HostController(base.Controller):
+    """Host info controller"""
+
+    @pecan.expose('json')
+    @base.Controller.api_version("1.0")
+    @exception.wrap_pecan_controller_exception
+    def get_all(self, **kwargs):
+        """Retrieve a list of hosts"""
+        context = pecan.request.context
+        policy.enforce(context, "host:get_all",
+                       action="host:get_all")
+        return self._get_host_collection(**kwargs)
+
+    def _get_host_collection(self, **kwargs):
+        context = pecan.request.context
+        limit = api_utils.validate_limit(kwargs.get('limit'))
+        sort_dir = api_utils.validate_sort_dir(kwargs.get('sort_dir', 'asc'))
+        sort_key = kwargs.get('sort_key', 'hostname')
+        expand = kwargs.get('expand')
+        filters = None
+        marker_obj = None
+        resource_url = kwargs.get('resource_url')
+        marker = kwargs.get('marker')
+        if marker:
+            marker_obj = objects.ComputeHost.get_by_uuid(context, marker)
+        hosts = objects.ComputeHost.list(context,
+                                         limit,
+                                         marker_obj,
+                                         sort_key,
+                                         sort_dir,
+                                         filters=filters)
+        return HostCollection.convert_with_links(hosts, limit,
+                                                 url=resource_url,
+                                                 expand=expand,
+                                                 sort_key=sort_key,
+                                                 sort_dir=sort_dir)
+
+    @pecan.expose('json')
+    @base.Controller.api_version("1.0")
+    @exception.wrap_pecan_controller_exception
+    def get_one(self, host_ident):
+        """Retrieve information about the given host.
+
+        :param host_ident: UUID or name of a host.
+        """
+        context = pecan.request.context
+        policy.enforce(context, "host:get", action="host:get")
+        host = _get_host(host_ident)
+        return view.format_host(pecan.request.host_url, host)

gyan/api/controllers/v1/ml_models.py

@@ -0,0 +1,289 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import shlex
+
+from oslo_log import log as logging
+from oslo_utils import strutils
+from oslo_utils import uuidutils
+import pecan
+import six
+
+from gyan.api.controllers import base
+from gyan.api.controllers import link
+from gyan.api.controllers.v1 import collection
+from gyan.api.controllers.v1.schemas import ml_models as schema
+from gyan.api.controllers.v1.views import ml_models_view as view
+from gyan.api import utils as api_utils
+from gyan.api import validation
+from gyan.common import consts
+from gyan.common import context as gyan_context
+from gyan.common import exception
+from gyan.common.i18n import _
+from gyan.common.policies import ml_model as policies
+from gyan.common import policy
+from gyan.common import utils
+import gyan.conf
+from gyan import objects
+
+CONF = gyan.conf.CONF
+LOG = logging.getLogger(__name__)
+
+
+def check_policy_on_ml_model(ml_model, action):
+    context = pecan.request.context
+    policy.enforce(context, action, ml_model, action=action)
+
+
+class MLModelCollection(collection.Collection):
+    """API representation of a collection of ml models."""
+
+    fields = {
+        'ml_models',
+        'next'
+    }
+
+    """A list containing ml models objects"""
+
+    def __init__(self, **kwargs):
+        super(MLModelCollection, self).__init__(**kwargs)
+        self._type = 'ml_models'
+
+    @staticmethod
+    def convert_with_links(rpc_ml_models, limit, url=None,
+                           expand=False, **kwargs):
+        context = pecan.request.context
+        collection = MLModelCollection()
+        collection.ml_models = \
+            [view.format_ml_model(context, url, p.as_dict())
+             for p in rpc_ml_models]
+        collection.next = collection.get_next(limit, url=url, **kwargs)
+        return collection
+
+
+class MLModelController(base.Controller):
+    """Controller for MLModels."""
+
+    _custom_actions = {
+        'train': ['POST'],
+        'deploy': ['GET'],
+        'undeploy': ['GET']
+    }
+
+    
+    @pecan.expose('json')
+    @exception.wrap_pecan_controller_exception
+    def get_all(self, **kwargs):
+        """Retrieve a list of ml models.
+
+        """
+        context = pecan.request.context
+        policy.enforce(context, "ml_model:get_all",
+                       action="ml_model:get_all")
+        return self._get_ml_models_collection(**kwargs)
+
+    def _get_ml_models_collection(self, **kwargs):
+        context = pecan.request.context
+        if utils.is_all_projects(kwargs):
+            policy.enforce(context, "ml_model:get_all_all_projects",
+                           action="ml_model:get_all_all_projects")
+            context.all_projects = True
+        kwargs.pop('all_projects', None)
+        limit = api_utils.validate_limit(kwargs.pop('limit', None))
+        sort_dir = api_utils.validate_sort_dir(kwargs.pop('sort_dir', 'asc'))
+        sort_key = kwargs.pop('sort_key', 'id')
+        resource_url = kwargs.pop('resource_url', None)
+        expand = kwargs.pop('expand', None)
+
+        ml_model_allowed_filters = ['name', 'status', 'project_id', 'user_id',
+                                     'type']
+        filters = {}
+        for filter_key in ml_model_allowed_filters:
+            if filter_key in kwargs:
+                policy_action = policies.MLMODEL % ('get_one:' + filter_key)
+                context.can(policy_action, might_not_exist=True)
+                filter_value = kwargs.pop(filter_key)
+                filters[filter_key] = filter_value
+        marker_obj = None
+        marker = kwargs.pop('marker', None)
+        if marker:
+            marker_obj = objects.ML_Model.get_by_uuid(context,
+                                                       marker)
+        if kwargs:
+            unknown_params = [str(k) for k in kwargs]
+            msg = _("Unknown parameters: %s") % ", ".join(unknown_params)
+            raise exception.InvalidValue(msg)
+
+        ml_models = objects.ML_Model.list(context,
+                                            limit,
+                                            marker_obj,
+                                            sort_key,
+                                            sort_dir,
+                                            filters=filters)
+        return MLModelCollection.convert_with_links(ml_models, limit,
+                                                      url=resource_url,
+                                                      expand=expand,
+                                                      sort_key=sort_key,
+                                                      sort_dir=sort_dir)
+
+    @pecan.expose('json')
+    @exception.wrap_pecan_controller_exception
+    def get_one(self, ml_model_ident, **kwargs):
+        """Retrieve information about the given ml_model.
+
+        :param ml_model_ident: UUID or name of a ml_model.
+        """
+        context = pecan.request.context
+        if utils.is_all_projects(kwargs):
+            policy.enforce(context, "ml_model:get_one_all_projects",
+                           action="ml_model:get_one_all_projects")
+            context.all_projects = True
+        ml_model = utils.get_ml_model(ml_model_ident)
+        check_policy_on_ml_model(ml_model.as_dict(), "ml_model:get_one")
+        if ml_model.node:
+            compute_api = pecan.request.compute_api
+            try:
+                ml_model = compute_api.ml_model_show(context, ml_model)
+            except exception.MLModelHostNotUp:
+                raise exception.ServerNotUsable
+
+        return view.format_ml_model(context, pecan.request.host_url,
+                                     ml_model.as_dict())
+
+    @base.Controller.api_version("1.0")
+    @pecan.expose('json')
+    @api_utils.enforce_content_types(['application/json'])
+    @exception.wrap_pecan_controller_exception
+    @validation.validate_query_param(pecan.request, schema.query_param_create)
+    @validation.validated(schema.ml_model_create)
+    def post(self, **ml_model_dict):
+        return self._do_post(**ml_model_dict)
+
+
+    def _do_post(self, **ml_model_dict):
+        """Create or run a new ml model.
+
+        :param ml_model_dict: a ml_model within the request body.
+        """
+        context = pecan.request.context
+        compute_api = pecan.request.compute_api
+        policy.enforce(context, "ml_model:create",
+                       action="ml_model:create")
+
+        ml_model_dict['project_id'] = context.project_id
+        ml_model_dict['user_id'] = context.user_id
+        name = ml_model_dict.get('name')
+        ml_model_dict['name'] = name
+        
+        ml_model_dict['status'] = consts.CREATING
+        extra_spec = {}
+        extra_spec['hints'] = ml_model_dict.get('hints', None)
+        new_ml_model = objects.ML_Model(context, **ml_model_dict)
+        new_ml_model.create(context)
+
+        compute_api.ml_model_create(context, new_ml_model, **kwargs)
+        # Set the HTTP Location Header
+        pecan.response.location = link.build_url('ml_models',
+                                                 new_ml_model.uuid)
+        pecan.response.status = 202
+        return view.format_ml_model(context, pecan.request.node_url,
+                                     new_ml_model.as_dict())
+
+    
+    @pecan.expose('json')
+    @exception.wrap_pecan_controller_exception
+    @validation.validated(schema.ml_model_update)
+    def patch(self, ml_model_ident, **patch):
+        """Update an existing ml model.
+
+        :param ml_model_ident: UUID or name of a ml model.
+        :param patch: a json PATCH document to apply to this ml model.
+        """
+        ml_model = utils.get_ml_model(ml_model_ident)
+        check_policy_on_ml_model(ml_model.as_dict(), "ml_model:update")
+        utils.validate_ml_model_state(ml_model, 'update')
+        context = pecan.request.context
+        compute_api = pecan.request.compute_api
+        ml_model = compute_api.ml_model_update(context, ml_model, patch)
+        return view.format_ml_model(context, pecan.request.node_url,
+                                     ml_model.as_dict())
+
+    
+    @pecan.expose('json')
+    @exception.wrap_pecan_controller_exception
+    @validation.validate_query_param(pecan.request, schema.query_param_delete)
+    def delete(self, ml_model_ident, force=False, **kwargs):
+        """Delete a ML Model.
+
+        :param ml_model_ident: UUID or Name of a ML Model.
+        :param force: If True, allow to force delete the ML Model.
+        """
+        context = pecan.request.context
+        ml_model = utils.get_ml_model(ml_model_ident)
+        check_policy_on_ml_model(ml_model.as_dict(), "ml_model:delete")
+        try:
+            force = strutils.bool_from_string(force, strict=True)
+        except ValueError:
+            bools = ', '.join(strutils.TRUE_STRINGS + strutils.FALSE_STRINGS)
+            raise exception.InvalidValue(_('Valid force values are: %s')
+                                         % bools)
+        stop = kwargs.pop('stop', False)
+        try:
+            stop = strutils.bool_from_string(stop, strict=True)
+        except ValueError:
+            bools = ', '.join(strutils.TRUE_STRINGS + strutils.FALSE_STRINGS)
+            raise exception.InvalidValue(_('Valid stop values are: %s')
+                                         % bools)
+        compute_api = pecan.request.compute_api
+        if not force:
+            utils.validate_ml_model_state(ml_model, 'delete')
+        ml_model.status = consts.DELETING
+        if ml_model.node:
+            compute_api.ml_model_delete(context, ml_model, force)
+        else:
+            ml_model.destroy(context)
+        pecan.response.status = 204
+
+
+    @pecan.expose('json')
+    @exception.wrap_pecan_controller_exception
+    def deploy(self, ml_model_ident, **kwargs):
+        """Deploy ML Model.
+
+        :param ml_model_ident: UUID or Name of a ML Model.
+        """
+        ml_model = utils.get_ml_model(ml_model_ident)
+        check_policy_on_ml_model(ml_model.as_dict(), "ml_model:deploy")
+        utils.validate_ml_model_state(ml_model, 'deploy')
+        LOG.debug('Calling compute.ml_model_deploy with %s',
+                  ml_model.uuid)
+        context = pecan.request.context
+        compute_api = pecan.request.compute_api
+        compute_api.ml_model_deploy(context, ml_model)
+        pecan.response.status = 202
+
+    @pecan.expose('json')
+    @exception.wrap_pecan_controller_exception
+    def undeploy(self, ml_model_ident, **kwargs):
+        """Undeploy ML Model.
+
+        :param ml_model_ident: UUID or Name of a ML Model.
+        """
+        ml_model = utils.get_ml_model(ml_model_ident)
+        check_policy_on_ml_model(ml_model.as_dict(), "ml_model:deploy")
+        utils.validate_ml_model_state(ml_model, 'undeploy')
+        LOG.debug('Calling compute.ml_model_deploy with %s',
+                  ml_model.uuid)
+        context = pecan.request.context
+        compute_api = pecan.request.compute_api
+        compute_api.ml_model_undeploy(context, ml_model)
+        pecan.response.status = 202

gyan/api/controllers/v1/schemas/ml_models.py

@@ -0,0 +1,49 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import copy
+
+from gyan.api.controllers.v1.schemas import parameter_types
+
+_ml_model_properties = {}
+
+ml_model_create = {
+    'type': 'object',
+    'properties': _ml_model_properties,
+    'required': ['name'],
+    'additionalProperties': False
+}
+
+
+query_param_create = {
+    'type': 'object',
+    'properties': {
+        'run': parameter_types.boolean_extended
+    },
+    'additionalProperties': False
+}
+
+ml_model_update = {
+    'type': 'object',
+    'properties': {},
+    'additionalProperties': False
+}
+
+query_param_delete = {
+    'type': 'object',
+    'properties': {
+        'force': parameter_types.boolean_extended,
+        'all_projects': parameter_types.boolean_extended,
+        'stop': parameter_types.boolean_extended
+    },
+    'additionalProperties': False
+}

gyan/api/controllers/v1/schemas/parameter_types.py

@@ -0,0 +1,97 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import copy
+import signal
+import sys
+import gyan.conf
+
+CONF = gyan.conf.CONF
+
+non_negative_integer = {
+    'type': ['integer', 'string'],
+    'pattern': '^[0-9]*$', 'minimum': 0
+}
+
+positive_integer = {
+    'type': ['integer', 'string'],
+    'pattern': '^[0-9]*$', 'minimum': 1
+}
+
+boolean_extended = {
+    'type': ['boolean', 'string'],
+    'enum': [True, 'True', 'TRUE', 'true', '1', 'ON', 'On', 'on',
+             'YES', 'Yes', 'yes',
+             False, 'False', 'FALSE', 'false', '0', 'OFF', 'Off', 'off',
+             'NO', 'No', 'no'],
+}
+
+boolean = {
+    'type': ['boolean', 'string'],
+    'enum': [True, 'True', 'true', False, 'False', 'false'],
+}
+
+ml_model_name = {
+    'type': ['string', 'null'],
+    'minLength': 2,
+    'maxLength': 255,
+    'pattern': '^[a-zA-Z0-9][a-zA-Z0-9_.-]+$'
+}
+
+hex_uuid = {
+    'type': 'string',
+    'maxLength': 32,
+    'minLength': 32,
+    'pattern': '^[a-fA-F0-9]*$'
+}
+
+
+labels = {
+    'type': ['object', 'null']
+}
+
+hints = {
+    'type': ['object', 'null']
+}
+hostname = {
+    'type': ['string', 'null'],
+    'minLength': 2,
+    'maxLength': 63
+}
+
+repo = {
+    'type': 'string',
+    'minLength': 2,
+    'maxLength': 255,
+    'pattern': '[a-zA-Z0-9][a-zA-Z0-9_.-]'
+}
+
+
+
+string_ps_args = {
+    'type': ['string'],
+    'pattern': '[a-zA-Z- ,+]*'
+}
+
+str_and_int = {
+    'type': ['string', 'integer', 'null'],
+}
+
+hostname = {
+    'type': 'string', 'minLength': 1, 'maxLength': 255,
+    # NOTE: 'host' is defined in "services" table, and that
+    # means a hostname. The hostname grammar in RFC952 does
+    # not allow for underscores in hostnames. However, this
+    # schema allows them, because it sometimes occurs in
+    # real systems.
+    'pattern': '^[a-zA-Z0-9-._]*$',
+}

gyan/api/controllers/v1/schemas/services.py

@@ -0,0 +1,50 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from gyan.api.controllers.v1.schemas import parameter_types
+
+query_param_enable = {
+    'type': 'object',
+    'properties': {
+        'host': parameter_types.hostname,
+        'binary': {
+            'type': 'string', 'minLength': 1, 'maxLength': 255,
+        },
+    },
+    'additionalProperties': False
+}
+
+query_param_disable = {
+    'type': 'object',
+    'properties': {
+        'host': parameter_types.hostname,
+        'binary': {
+            'type': 'string', 'minLength': 1, 'maxLength': 255,
+        },
+        'disabled_reason': {
+            'type': 'string', 'minLength': 1, 'maxLength': 255,
+        },
+    },
+    'additionalProperties': False
+}
+
+query_param_force_down = {
+    'type': 'object',
+    'properties': {
+        'host': parameter_types.hostname,
+        'binary': {
+            'type': 'string', 'minLength': 1, 'maxLength': 255,
+        },
+        'forced_down': parameter_types.boolean
+    },
+    'additionalProperties': False
+}

gyan/api/controllers/v1/views/hosts_view.py

@@ -0,0 +1,43 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import itertools
+
+from gyan.api.controllers import link
+
+
+_basic_keys = (
+    'id',
+    'hostname',
+    'type',
+    'status'
+)
+
+
+def format_host(url, host):
+    def transform(key, value):
+        if key not in _basic_keys:
+            return
+        if key == 'id':
+            yield ('id', value)
+            yield ('links', [link.make_link(
+                'self', url, 'hosts', value),
+                link.make_link(
+                    'bookmark', url,
+                    'hosts', value,
+                    bookmark=True)])
+        else:
+            yield (key, value)
+
+    return dict(itertools.chain.from_iterable(
+        transform(k, v) for k, v in host.as_dict().items()))

gyan/api/controllers/v1/views/ml_models_view.py

@@ -0,0 +1,55 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import itertools
+
+from gyan.api.controllers import link
+from gyan.common.policies import ml_model as policies
+
+_basic_keys = (
+    'uuid',
+    'user_id',
+    'project_id',
+    'name',
+    'url',
+    'status',
+    'status_reason',
+    'task_state',
+    'labels',
+    'host',
+    'status_detail'
+)
+
+
+def format_ml_model(context, url, ml_model):
+    def transform(key, value):
+        if key not in _basic_keys:
+            return
+        # strip the key if it is not allowed by policy
+        policy_action = policies.ML_MODEL % ('get_one:%s' % key)
+        if not context.can(policy_action, fatal=False, might_not_exist=True):
+            return
+        if key == 'uuid':
+            yield ('uuid', value)
+            if url:
+                yield ('links', [link.make_link(
+                    'self', url, 'ml_models', value),
+                    link.make_link(
+                        'bookmark', url,
+                        'ml_models', value,
+                        bookmark=True)])
+        else:
+            yield (key, value)
+
+    return dict(itertools.chain.from_iterable(
+        transform(k, v) for k, v in ml_model.items()))

gyan/api/controllers/versions.py

@@ -0,0 +1,145 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from webob import exc
+
+from gyan.common.i18n import _
+
+# NOTE(tbh):  v1.0 is reserved to indicate Ocata's API, but is not presently
+#             supported by the API service. All changes between Ocata and the
+#             point where we added microversioning are considered backwards-
+#             compatible, but are not specifically discoverable at this time.
+#
+#             The v1.1 version indicates this "initial" version as being
+#             different from Ocata (v1.0), and includes the following changes:
+#
+#             Add details of new api versions here:
+
+#
+# For each newly added microversion change, update the API version history
+# string below with a one or two line description. Also update
+# rest_api_version_history.rst for extra information on microversion.
+REST_API_VERSION_HISTORY = """REST API Version History:
+
+    * 1.1 - Initial version
+"""
+
+BASE_VER = '1.1'
+CURRENT_MAX_VER = '1.1'
+
+
+class Version(object):
+    """API Version object."""
+
+    string = 'OpenStack-API-Version'
+    """HTTP Header string carrying the requested version"""
+
+    min_string = 'OpenStack-API-Minimum-Version'
+    """HTTP response header"""
+
+    max_string = 'OpenStack-API-Maximum-Version'
+    """HTTP response header"""
+
+    service_string = 'ml'
+
+    def __init__(self, headers, default_version, latest_version,
+                 from_string=None):
+        """Create an API Version object from the supplied headers.
+
+        :param headers: webob headers
+        :param default_version: version to use if not specified in headers
+        :param latest_version: version to use if latest is requested
+        :param from_string: create the version from string not headers
+        :raises: webob.HTTPNotAcceptable
+        """
+        if from_string:
+            (self.major, self.minor) = tuple(int(i)
+                                             for i in from_string.split('.'))
+
+        else:
+            (self.major, self.minor) = Version.parse_headers(headers,
+                                                             default_version,
+                                                             latest_version)
+
+    def __repr__(self):
+        return '%s.%s' % (self.major, self.minor)
+
+    @staticmethod
+    def parse_headers(headers, default_version, latest_version):
+        """Determine the API version requested based on the headers supplied.
+
+        :param headers: webob headers
+        :param default_version: version to use if not specified in headers
+        :param latest_version: version to use if latest is requested
+        :returns: a tuple of (major, minor) version numbers
+        :raises: webob.HTTPNotAcceptable
+        """
+
+        version_hdr = headers.get(Version.string, default_version)
+
+        try:
+            version_service, version_str = version_hdr.split()
+        except ValueError:
+            raise exc.HTTPNotAcceptable(_(
+                "Invalid service type for %s header") % Version.string)
+
+        if version_str.lower() == 'latest':
+            version_service, version_str = latest_version.split()
+
+        if version_service != Version.service_string:
+            raise exc.HTTPNotAcceptable(_(
+                "Invalid service type for %s header") % Version.string)
+        try:
+            version = tuple(int(i) for i in version_str.split('.'))
+        except ValueError:
+            version = ()
+
+        if len(version) != 2:
+            raise exc.HTTPNotAcceptable(_(
+                "Invalid value for %s header") % Version.string)
+        return version
+
+    def is_null(self):
+        return self.major == 0 and self.minor == 0
+
+    def matches(self, start_version, end_version):
+        if self.is_null():
+            raise ValueError
+
+        return start_version <= self <= end_version
+
+    def __lt__(self, other):
+        if self.major < other.major:
+            return True
+        if self.major == other.major and self.minor < other.minor:
+            return True
+        return False
+
+    def __gt__(self, other):
+        if self.major > other.major:
+            return True
+        if self.major == other.major and self.minor > other.minor:
+            return True
+        return False
+
+    def __eq__(self, other):
+        return self.major == other.major and self.minor == other.minor
+
+    def __le__(self, other):
+        return self < other or self == other
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __ge__(self, other):
+        return self > other or self == other

gyan/api/hooks.py

@@ -0,0 +1,114 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+from pecan import hooks
+
+from gyan.common import context
+from gyan.compute import api as compute_api
+import gyan.conf
+
+CONF = gyan.conf.CONF
+
+
+class ContextHook(hooks.PecanHook):
+    """Configures a request context and attaches it to the request.
+
+    The following HTTP request headers are used:
+
+    X-User-Name:
+        Used for context.user_name.
+
+    X-User-Id:
+        Used for context.user_id.
+
+    X-Project-Name:
+        Used for context.project.
+
+    X-Project-Id:
+        Used for context.project_id.
+
+    X-Auth-Token:
+        Used for context.auth_token.
+
+    X-Roles:
+        Used for context.roles.
+    """
+
+    def before(self, state):
+        headers = state.request.headers
+        user_name = headers.get('X-User-Name')
+        user_id = headers.get('X-User-Id')
+        project = headers.get('X-Project-Name')
+        project_id = headers.get('X-Project-Id')
+        domain_id = headers.get('X-User-Domain-Id')
+        domain_name = headers.get('X-User-Domain-Name')
+        auth_token = headers.get('X-Auth-Token')
+        roles = headers.get('X-Roles', '').split(',')
+        auth_token_info = state.request.environ.get('keystone.token_info')
+
+        state.request.context = context.make_context(
+            auth_token=auth_token,
+            auth_token_info=auth_token_info,
+            user_name=user_name,
+            user_id=user_id,
+            project_name=project,
+            project_id=project_id,
+            domain_id=domain_id,
+            domain_name=domain_name,
+            roles=roles)
+
+
+class RPCHook(hooks.PecanHook):
+    """Attach the rpcapi object to the request so controllers can get to it."""
+
+    def before(self, state):
+        context = state.request.context
+        state.request.compute_api = compute_api.API(context)
+
+
+class NoExceptionTracebackHook(hooks.PecanHook):
+    """Workaround rpc.common: deserialize_remote_exception.
+
+    deserialize_remote_exception builds rpc exception traceback into error
+    message which is then sent to the client. Such behavior is a security
+    concern so this hook is aimed to cut-off traceback from the error message.
+    """
+    # NOTE(tbh): 'after' hook used instead of 'on_error' because
+    # 'on_error' never fired for wsme+pecan pair. wsme @wsexpose decorator
+    # catches and handles all the errors, so 'on_error' dedicated for unhandled
+    # exceptions never fired.
+    def after(self, state):
+        # Omit empty body. Some errors may not have body at this level yet.
+        if not state.response.body:
+            return
+
+        # Do nothing if there is no error.
+        if 200 <= state.response.status_int < 400:
+            return
+
+        json_body = state.response.json
+        # Do not remove traceback when server in debug mode (except 'Server'
+        # errors when 'debuginfo' will be used for traces).
+        if CONF.debug and json_body.get('faultcode') != 'Server':
+            return
+
+        title = json_body.get('title')
+        traceback_marker = 'Traceback (most recent call last):'
+        if title and (traceback_marker in title):
+            # Cut-off traceback.
+            title = title.split(traceback_marker, 1)[0]
+            # Remove trailing newlines and spaces if any.
+            json_body['title'] = title.rstrip()
+            # Replace the whole json. Cannot change original one beacause it's
+            # generated on the fly.
+            state.response.json = json_body

gyan/api/http_error.py

@@ -0,0 +1,69 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import six
+
+from oslo_serialization import jsonutils as json
+from webob import exc
+
+
+class HTTPNotAcceptableAPIVersion(exc.HTTPNotAcceptable):
+    # subclass of :class:`~HTTPNotAcceptable`
+    #
+    # This indicates the resource identified by the request is only
+    # capable of generating response entities which have content
+    # characteristics not acceptable according to the accept headers
+    # sent in the request.
+    #
+    # code: 406, title: Not Acceptable
+    #
+    # differences from webob.exc.HTTPNotAcceptable:
+    #
+    # - additional max and min version parameters
+    # - additional error info for code, title, and links
+    code = 406
+    title = 'Not Acceptable'
+    max_version = ''
+    min_version = ''
+
+    def __init__(self, detail=None, headers=None, comment=None,
+                 body_template=None, max_version='', min_version='', **kwargs):
+
+        super(HTTPNotAcceptableAPIVersion, self).__init__(
+            detail=detail, headers=headers, comment=comment,
+            body_template=body_template, **kwargs)
+
+        self.max_version = max_version
+        self.min_version = min_version
+
+    def __call__(self, environ, start_response):
+        for err_str in self.app_iter:
+            err = {}
+            try:
+                err = json.loads(err_str.decode('utf-8'))
+            except ValueError:
+                pass
+
+            links = {'rel': 'help', 'href': 'https://developer.openstack.org'
+                     '/api-guide/compute/microversions.html'}
+
+            err['max_version'] = self.max_version
+            err['min_version'] = self.min_version
+            err['code'] = "gyan.microversion-unsupported"
+            err['links'] = [links]
+            err['title'] = "Requested microversion is unsupported"
+
+        self.app_iter = [six.b(json.dump_as_bytes(err))]
+        self.headers['Content-Length'] = str(len(self.app_iter[0]))
+
+        return super(HTTPNotAcceptableAPIVersion, self).__call__(
+            environ, start_response)

gyan/api/middleware/__init__.py

@@ -0,0 +1,21 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from gyan.api.middleware import auth_token
+from gyan.api.middleware import parsable_error
+
+
+AuthTokenMiddleware = auth_token.AuthTokenMiddleware
+ParsableErrorMiddleware = parsable_error.ParsableErrorMiddleware
+
+__all__ = ('AuthTokenMiddleware',
+           'ParsableErrorMiddleware')

gyan/api/middleware/auth_token.py

@@ -0,0 +1,71 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import re
+
+from keystonemiddleware import auth_token
+from oslo_log import log
+
+from gyan.common import exception
+from gyan.common.i18n import _
+from gyan.common import utils
+
+LOG = log.getLogger(__name__)
+
+
+class AuthTokenMiddleware(auth_token.AuthProtocol):
+    """A wrapper on Keystone auth_token middleware.
+
+    Does not perform verification of authentication tokens
+    for public routes in the API.
+
+    """
+
+    def __init__(self, app, conf, public_api_routes=None):
+        if public_api_routes is None:
+            public_api_routes = []
+        route_pattern_tpl = '%s(\.json)?$'
+
+        try:
+            self.public_api_routes = [re.compile(route_pattern_tpl % route_tpl)
+                                      for route_tpl in public_api_routes]
+        except re.error as e:
+            msg = _('Cannot compile public API routes: %s') % e
+
+            LOG.error(msg)
+            raise exception.ConfigInvalid(error_msg=msg)
+
+        super(AuthTokenMiddleware, self).__init__(app, conf)
+
+    def __call__(self, env, start_response):
+        path = utils.safe_rstrip(env.get('PATH_INFO'), '/')
+
+        # The information whether the API call is being performed against the
+        # public API is required for some other components. Saving it to the
+        # WSGI environment is reasonable thereby.
+        env['is_public_api'] = any([re.match(pattern, path)
+                                    for pattern in self.public_api_routes])
+
+        if env['is_public_api']:
+            return self._app(env, start_response)
+
+        return super(AuthTokenMiddleware, self).__call__(env, start_response)
+
+    @classmethod
+    def factory(cls, global_config, **local_conf):
+        public_routes = local_conf.get('acl_public_routes', '')
+        public_api_routes = [path.strip() for path in public_routes.split(',')]
+
+        def _factory(app):
+            return cls(app, global_config, public_api_routes=public_api_routes)
+
+        return _factory

gyan/api/middleware/parsable_error.py

@@ -0,0 +1,99 @@
+# Copyright ? 2012 New Dream Network, LLC (DreamHost)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""
+Middleware to replace the plain text message body of an error
+response with one formatted so the client can parse it.
+
+Based on pecan.middleware.errordocument
+"""
+
+import six
+
+from oslo_serialization import jsonutils as json
+from gyan.common.i18n import _
+
+
+class ParsableErrorMiddleware(object):
+    """Replace error body with something the client can parse."""
+
+    def __init__(self, app):
+        self.app = app
+
+    def __call__(self, environ, start_response):
+        # Request for this state, modified by replace_start_response()
+        # and used when an error is being reported.
+        state = {}
+
+        def replacement_start_response(status, headers, exc_info=None):
+            """Overrides the default response to make errors parsable."""
+            try:
+                status_code = int(status.split(' ')[0])
+                state['status_code'] = status_code
+            except (ValueError, TypeError):  # pragma: nocover
+                raise Exception(_(
+                    'ErrorDocumentMiddleware received an invalid '
+                    'status %s') % status)
+            else:
+                if (state['status_code'] // 100) not in (2, 3):
+                    # Remove some headers so we can replace them later
+                    # when we have the full error message and can
+                    # compute the length.
+                    headers = [(h, v)
+                               for (h, v) in headers
+                               if h not in ('Content-Length', 'Content-Type')
+                               ]
+                # Save the headers in case we need to modify them.
+                state['headers'] = headers
+                return start_response(status, headers, exc_info)
+
+        app_iter = self.app(environ, replacement_start_response)
+
+        if (state['status_code'] // 100) not in (2, 3):
+            errs = []
+            for err_str in app_iter:
+                err = {}
+                try:
+                    err = json.loads(err_str.decode('utf-8'))
+                except ValueError:
+                    pass
+
+                if 'title' in err and 'description' in err:
+                    title = err['title']
+                    desc = err['description']
+                else:
+                    title = ''
+                    desc = ''
+
+                error_code = err['faultstring'].lower() \
+                    if 'faultstring' in err else ''
+                # 'ml-infra' is the service-name. The general form of the
+                # code is service-name.error-code.
+                code = '.'.join(['ml-infra', error_code])
+
+                errs.append({
+                    'request_id': '',
+                    'code': code,
+                    'status': state['status_code'],
+                    'title': title,
+                    'detail': desc,
+                    'links': []
+                })
+
+            body = [six.b(json.dumps({'errors': errs}))]
+
+            state['headers'].append(('Content-Type', 'application/json'))
+            state['headers'].append(('Content-Length', str(len(body[0]))))
+        else:
+            body = app_iter
+        return body

gyan/api/servicegroup.py

@@ -0,0 +1,37 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_utils import timeutils
+
+import gyan.conf
+from gyan import objects
+
+CONF = gyan.conf.CONF
+
+
+class ServiceGroup(object):
+    def __init__(self):
+        self.service_down_time = CONF.service_down_time
+
+    def service_is_up(self, member):
+        if not isinstance(member, objects.GyanService):
+            raise TypeError
+        if member.forced_down:
+            return False
+
+        last_heartbeat = (member.last_seen_up or
+                          member.updated_at or member.created_at)
+        now = timeutils.utcnow()
+        elapsed = timeutils.delta_seconds(last_heartbeat, now)
+        is_up = abs(elapsed) <= self.service_down_time
+        return is_up

gyan/api/utils.py

@@ -0,0 +1,116 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import functools
+from oslo_utils import uuidutils
+import pecan
+
+from gyan.api.controllers import versions
+from gyan.common import exception
+from gyan.common.i18n import _
+import gyan.conf
+from gyan import objects
+
+CONF = gyan.conf.CONF
+
+
+def string_or_none(value):
+    if value in [None, 'None']:
+        return None
+    else:
+        return value
+
+
+def validate_limit(limit):
+    try:
+        if limit is not None and int(limit) <= 0:
+            raise exception.InvalidValue(_("Limit must be positive integer"))
+    except ValueError:
+        raise exception.InvalidValue(_("Limit must be positive integer"))
+
+    if limit is not None:
+        return min(CONF.api.max_limit, int(limit))
+    else:
+        return CONF.api.max_limit
+
+
+def validate_sort_dir(sort_dir):
+    if sort_dir not in ['asc', 'desc']:
+        raise exception.InvalidValue(_("Invalid sort direction: %s. "
+                                       "Acceptable values are "
+                                       "'asc' or 'desc'") % sort_dir)
+    return sort_dir
+
+
+def get_resource(resource, resource_ident):
+    """Get the resource from the uuid or logical name.
+
+    :param resource: the resource type.
+    :param resource_ident: the UUID or logical name of the resource.
+
+    :returns: The resource.
+    """
+    resource = getattr(objects, resource)
+    context = pecan.request.context
+    if context.is_admin:
+        context.all_projects = True
+    if uuidutils.is_uuid_like(resource_ident):
+        return resource.get_by_uuid(context, resource_ident)
+
+    return resource.get_by_name(context, resource_ident)
+
+
+def _do_enforce_content_types(pecan_req, valid_content_types):
+    """Content type enforcement
+
+    Check to see that content type in the request is one of the valid
+    types passed in by our caller.
+    """
+    if pecan_req.content_type not in valid_content_types:
+        m = (
+            "Unexpected content type: {type}. Expected content types "
+            "are: {expected}"
+        ).format(
+            type=pecan_req.content_type.decode('utf-8'),
+            expected=valid_content_types
+        )
+        pecan.abort(415, m)
+
+
+def enforce_content_types(valid_content_types):
+    """Decorator handling content type enforcement on behalf of REST verbs."""
+
+    def content_types_decorator(fn):
+
+        @functools.wraps(fn)
+        def content_types_enforcer(inst, *args, **kwargs):
+            _do_enforce_content_types(pecan.request, valid_content_types)
+            return fn(inst, *args, **kwargs)
+
+        return content_types_enforcer
+
+    return content_types_decorator
+
+
+def version_check(action, version):
+    """Check whether the current version supports the operation.
+
+    :param action: Operations to be executed.
+    :param version: The minimum version required to perform the operation.
+
+    """
+    req_version = pecan.request.version
+    min_version = versions.Version('', '', '', version)
+    if req_version < min_version:
+        raise exception.InvalidParamInVersion(param=action,
+                                              req_version=req_version,
+                                              min_version=min_version)

gyan/api/validation/__init__.py

@@ -0,0 +1,57 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import functools
+
+from gyan.api.validation import validators
+
+
+def validated(request_body_schema):
+    """Register a schema to validate a resource reference.
+
+    Registered schema will be used for validating a request body just before
+    API method execution.
+
+    :param request_body_schema: a schema to validate the resource reference
+    """
+    schema_validator = validators.SchemaValidator(request_body_schema,
+                                                  is_body=True)
+
+    def add_validator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            schema_validator.validate(kwargs)
+            return func(*args, **kwargs)
+        return wrapper
+    return add_validator
+
+
+def validate_query_param(req, query_param_schema):
+    """Register a schema to validate a resource reference.
+
+    Registered schema will be used for validating a request query params
+    just before API method execution.
+
+    :param req: the request object
+    :param query_param_schema: a schema to validate the resource reference
+    """
+
+    schema_validator = validators.SchemaValidator(query_param_schema,
+                                                  is_body=False)
+
+    def add_validator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            schema_validator.validate(req.params.mixed())
+            return func(*args, **kwargs)
+        return wrapper
+    return add_validator

gyan/api/validation/validators.py

@@ -0,0 +1,80 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import jsonschema
+import six
+
+from gyan.common import exception
+from gyan.common.i18n import _
+
+
+class SchemaValidator(object):
+    """Resource reference validator class."""
+
+    validator_org = jsonschema.Draft4Validator
+
+    def __init__(self, schema, is_body=True):
+        self.is_body = is_body
+        validators = {
+            'minimum': self._validate_minimum,
+            'maximum': self._validate_maximum
+        }
+        validator_cls = jsonschema.validators.extend(self.validator_org,
+                                                     validators)
+        fc = jsonschema.FormatChecker()
+        self.validator = validator_cls(schema, format_checker=fc)
+
+    def validate(self, *args, **kwargs):
+        try:
+            self.validator.validate(*args, **kwargs)
+        except jsonschema.ValidationError as ex:
+            if len(ex.path) > 0:
+                if self.is_body:
+                    detail = _("Invalid input for field '%(path)s'."
+                               "Value: '%(value)s'. %(message)s")
+                else:
+                    detail = _("Invalid input for query parameters "
+                               "'%(path)s'. Value: '%(value)s'. %(message)s")
+                detail = detail % {
+                    'path': ex.path.pop(), 'value': ex.instance,
+                    'message': six.text_type(ex)
+                }
+            else:
+                detail = six.text_type(ex)
+            raise exception.SchemaValidationError(detail=detail)
+
+    def _number_from_str(self, instance):
+        if isinstance(instance, float) or isinstance(instance, int):
+            return instance
+
+        try:
+            value = int(instance)
+        except (ValueError, TypeError):
+            try:
+                value = float(instance)
+            except (ValueError, TypeError):
+                return None
+        return value
+
+    def _validate_minimum(self, validator, minimum, instance, schema):
+        instance = self._number_from_str(instance)
+        if instance is None:
+            return
+        return self.validator_org.VALIDATORS['minimum'](validator, minimum,
+                                                        instance, schema)
+
+    def _validate_maximum(self, validator, maximum, instance, schema):
+        instance = self._number_from_str(instance)
+        if instance is None:
+            return
+        return self.validator_org.VALIDATORS['maximum'](validator, maximum,
+                                                        instance, schema)

gyan/api/versioned_method.py

@@ -0,0 +1,33 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+class VersionedMethod(object):
+
+    def __init__(self, name, start_version, end_version, func):
+        """Versioning information for a single method
+
+        @name: Name of the method
+        @start_version: Minimum acceptable version
+        @end_version: Maximum acceptable_version
+        @func: Method to call
+
+        Minimum and maximums are inclusive
+        """
+        self.name = name
+        self.start_version = start_version
+        self.end_version = end_version
+        self.func = func
+
+    def __str__(self):
+        return ("Version Method %s: min: %s, max: %s"
+                % (self.name, self.start_version, self.end_version))

gyan/api/wsgi.py

@@ -0,0 +1,36 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sys
+
+from oslo_log import log
+
+from gyan.api import app
+from gyan.common import profiler
+from gyan.common import service
+import gyan.conf
+
+
+CONF = gyan.conf.CONF
+LOG = log.getLogger(__name__)
+
+
+def init_application():
+    # Initialize the oslo configuration library and logging
+    service.prepare_service(sys.argv)
+    # NOTE:(tbh) change localhost to CONF.host
+    profiler.setup('gyan-api', "localhost")
+
+    LOG.debug("Configuration:")
+    CONF.log_opt_values(LOG, log.DEBUG)
+
+    return app.load_app()

gyan/cmd/__init__.py

@@ -0,0 +1,17 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# NOTE(tbh): we monkey patch all eventlet services for easier tracking/debug
+
+import eventlet
+
+eventlet.monkey_patch()

gyan/cmd/api.py

@@ -0,0 +1,45 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""The gyan Service API."""
+
+import sys
+
+from gyan.common import profiler
+from gyan.common import service as gyan_service
+import gyan.conf
+
+CONF = gyan.conf.CONF
+
+
+def main():
+    # Parse config file and command line options, then start logging
+    gyan_service.prepare_service(sys.argv)
+
+    # Enable object backporting via the conductor
+    # TODO(tbh): Uncomment after rpc services are implemented
+    # base.gyanObject.indirection_api = base.gyanObjectIndirectionAPI()
+
+    # Setup OSprofiler for WSGI service
+    profiler.setup('gyan-api', CONF.api.host_ip)
+
+    # Build and start the WSGI app
+    launcher = gyan_service.process_launcher()
+    server = gyan_service.WSGIService(
+        'gyan_api',
+        CONF.api.enable_ssl_api
+    )
+    launcher.launch_service(server, workers=server.workers)
+    launcher.wait()
+
+if __name__ == '__main__':
+    sys.exit(main())

gyan/cmd/compute.py

@@ -0,0 +1,47 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import shlex
+import sys
+
+from oslo_log import log as logging
+from oslo_privsep import priv_context
+from oslo_service import service
+
+from gyan.common import rpc_service
+from gyan.common import service as gyan_service
+from gyan.common import utils
+import gyan.conf
+
+CONF = gyan.conf.CONF
+LOG = logging.getLogger(__name__)
+
+
+def main():
+    gyan_service.prepare_service(sys.argv)
+
+    LOG.info('Starting server in PID %s', os.getpid())
+    CONF.log_opt_values(LOG, logging.DEBUG)
+
+    CONF.import_opt('topic', 'gyan.conf.compute', group='compute')
+    CONF.import_opt('host', 'gyan.conf.compute', group='compute')
+    
+
+    from gyan.compute import manager as compute_manager
+    endpoints = [
+        compute_manager.Manager(),
+    ]
+    server = rpc_service.Service.create(CONF.compute.topic, CONF.compute.host,
+                                        endpoints, binary='gyan-compute')
+    launcher = service.launch(CONF, server, restart_method='mutate')
+    launcher.wait()

gyan/cmd/db_manage.py

@@ -0,0 +1,67 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Starter script for gyan-db-manage."""
+
+from oslo_config import cfg
+
+from gyan.db import migration
+
+
+CONF = cfg.CONF
+
+
+def do_version():
+    print('Current DB revision is %s' % migration.version())
+
+
+def do_upgrade():
+    migration.upgrade(CONF.command.revision)
+
+
+def do_stamp():
+    migration.stamp(CONF.command.revision)
+
+
+def do_revision():
+    migration.revision(message=CONF.command.message,
+                       autogenerate=CONF.command.autogenerate)
+
+
+def add_command_parsers(subparsers):
+    parser = subparsers.add_parser('version')
+    parser.set_defaults(func=do_version)
+
+    parser = subparsers.add_parser('upgrade')
+    parser.add_argument('revision', nargs='?')
+    parser.set_defaults(func=do_upgrade)
+
+    parser = subparsers.add_parser('stamp')
+    parser.add_argument('revision')
+    parser.set_defaults(func=do_stamp)
+
+    parser = subparsers.add_parser('revision')
+    parser.add_argument('-m', '--message')
+    parser.add_argument('--autogenerate', action='store_true')
+    parser.set_defaults(func=do_revision)
+
+
+def main():
+    command_opt = cfg.SubCommandOpt('command',
+                                    title='Command',
+                                    help='Available commands',
+                                    handler=add_command_parsers)
+    CONF.register_cli_opt(command_opt)
+
+    CONF(project='gyan')
+    CONF.command.func()

gyan/common/config.py

@@ -0,0 +1,54 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_middleware import cors
+
+from gyan.common import rpc
+import gyan.conf
+from gyan import version
+
+
+def parse_args(argv, default_config_files=None):
+    rpc.set_defaults(control_exchange='gyan')
+    gyan.conf.CONF(argv[1:],
+                  project='gyan',
+                  version=version.version_info.release_string(),
+                  default_config_files=default_config_files)
+    rpc.init(gyan.conf.CONF)
+
+
+def set_config_defaults():
+    """This method updates all configuration default values."""
+    set_cors_middleware_defaults()
+
+
+def set_cors_middleware_defaults():
+    """Update default configuration options for oslo.middleware."""
+    cors.set_defaults(
+        allow_headers=['X-Auth-Token',
+                       'X-Identity-Status',
+                       'X-Roles',
+                       'X-Service-Catalog',
+                       'X-User-Id',
+                       'X-Project-Id',
+                       'X-OpenStack-Request-ID',
+                       'X-Server-Management-Url'],
+        expose_headers=['X-Auth-Token',
+                        'X-Subject-Token',
+                        'X-Service-Token',
+                        'X-OpenStack-Request-ID',
+                        'X-Server-Management-Url'],
+        allow_methods=['GET',
+                       'PUT',
+                       'POST',
+                       'DELETE',
+                       'PATCH'])

gyan/common/consts.py

@@ -0,0 +1,17 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ALLOCATED = 'allocated'
+CREATED = 'created'
+UNDEPLOYED = 'undeployed'
+DEPLOYED = 'deployed'

gyan/common/context.py

@@ -0,0 +1,180 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import functools
+
+import copy
+from oslo_context import context
+from oslo_utils import timeutils
+import six
+
+from gyan.common import exception
+from gyan.common import policy
+
+
+class RequestContext(context.RequestContext):
+    """Extends security contexts from the OpenStack common library."""
+
+    def __init__(self, auth_token=None, domain_id=None,
+                 domain_name=None, user_name=None, user_id=None,
+                 user_domain_name=None, user_domain_id=None,
+                 project_name=None, project_id=None, roles=None,
+                 is_admin=None, read_only=False, show_deleted=False,
+                 request_id=None, trust_id=None, auth_token_info=None,
+                 all_projects=False, password=None, timestamp=None, **kwargs):
+        """Stores several additional request parameters:
+
+        :param domain_id: The ID of the domain.
+        :param domain_name: The name of the domain.
+        :param user_domain_id: The ID of the domain to
+                               authenticate a user against.
+        :param user_domain_name: The name of the domain to
+                                 authenticate a user against.
+
+        """
+        super(RequestContext, self).__init__(auth_token=auth_token,
+                                             user_id=user_name,
+                                             project_id=project_name,
+                                             is_admin=is_admin,
+                                             read_only=read_only,
+                                             show_deleted=show_deleted,
+                                             request_id=request_id,
+                                             roles=roles)
+
+        self.user_name = user_name
+        self.user_id = user_id
+        self.project_name = project_name
+        self.project_id = project_id
+        self.domain_id = domain_id
+        self.domain_name = domain_name
+        self.user_domain_id = user_domain_id
+        self.user_domain_name = user_domain_name
+        self.auth_token_info = auth_token_info
+        self.trust_id = trust_id
+        self.all_projects = all_projects
+        self.password = password
+        if is_admin is None:
+            self.is_admin = policy.check_is_admin(self)
+        else:
+            self.is_admin = is_admin
+
+        if not timestamp:
+            timestamp = timeutils.utcnow()
+        if isinstance(timestamp, six.string_types):
+            timestamp = timeutils.parse_strtime(timestamp)
+        self.timestamp = timestamp
+
+    def to_dict(self):
+        value = super(RequestContext, self).to_dict()
+        value.update({'auth_token': self.auth_token,
+                      'domain_id': self.domain_id,
+                      'domain_name': self.domain_name,
+                      'user_domain_id': self.user_domain_id,
+                      'user_domain_name': self.user_domain_name,
+                      'user_name': self.user_name,
+                      'user_id': self.user_id,
+                      'project_name': self.project_name,
+                      'project_id': self.project_id,
+                      'is_admin': self.is_admin,
+                      'read_only': self.read_only,
+                      'roles': self.roles,
+                      'show_deleted': self.show_deleted,
+                      'request_id': self.request_id,
+                      'trust_id': self.trust_id,
+                      'auth_token_info': self.auth_token_info,
+                      'password': self.password,
+                      'all_projects': self.all_projects,
+                      'timestamp': timeutils.strtime(self.timestamp) if
+                      hasattr(self, 'timestamp') else None
+                      })
+        return value
+
+    def to_policy_values(self):
+        policy = super(RequestContext, self).to_policy_values()
+        policy['is_admin'] = self.is_admin
+        return policy
+
+    @classmethod
+    def from_dict(cls, values):
+        return cls(**values)
+
+    def elevated(self):
+        """Return a version of this context with admin flag set."""
+        context = copy.copy(self)
+        # context.roles must be deepcopied to leave original roles
+        # without changes
+        context.roles = copy.deepcopy(self.roles)
+        context.is_admin = True
+
+        if 'admin' not in context.roles:
+            context.roles.append('admin')
+
+        return context
+
+    def can(self, action, target=None, fatal=True, might_not_exist=False):
+        """Verifies that the given action is valid on the target in this context.
+
+        :param action: string representing the action to be checked.
+        :param target: dictionary representing the object of the action
+            for object creation this should be a dictionary representing the
+            location of the object e.g. ``{'project_id': context.project_id}``.
+            If None, then this default target will be considered:
+            {'project_id': self.project_id, 'user_id': self.user_id}
+        :param fatal: if False, will return False when an
+            exception.NotAuthorized occurs.
+        :param might_not_exist: If True the policy check is skipped (and the
+            function returns True) if the specified policy does not exist.
+            Defaults to false.
+
+        :raises gyan.common.exception.NotAuthorized: if verification fails and
+            fatal is True.
+
+        :return: returns a non-False value (not necessarily "True") if
+            authorized and False if not authorized and fatal is False.
+        """
+        if target is None:
+            target = {'project_id': self.project_id,
+                      'user_id': self.user_id}
+
+        try:
+            return policy.authorize(self, action, target,
+                                    might_not_exist=might_not_exist)
+        except exception.NotAuthorized:
+            if fatal:
+                raise
+            return False
+
+
+def make_context(*args, **kwargs):
+    return RequestContext(*args, **kwargs)
+
+
+def get_admin_context(show_deleted=False, all_projects=False):
+    """Create an administrator context.
+
+    :param show_deleted: if True, will show deleted items when query db
+    """
+    context = RequestContext(user_id=None,
+                             project=None,
+                             is_admin=True,
+                             show_deleted=show_deleted,
+                             all_projects=all_projects)
+    return context
+
+
+def set_context(func):
+    @functools.wraps(func)
+    def handler(self, ctx):
+        if ctx is None:
+            ctx = get_admin_context(all_projects=True)
+        func(self, ctx)
+    return handler

gyan/common/exception.py

@@ -0,0 +1,485 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Gyan base exception handling.
+
+Includes decorator for re-raising Gyan-type exceptions.
+
+"""
+
+import functools
+import inspect
+import re
+import sys
+
+from keystoneclient import exceptions as keystone_exceptions
+from oslo_config import cfg
+from oslo_log import log as logging
+from oslo_utils import excutils
+from oslo_utils import uuidutils
+import pecan
+import six
+from webob import util as woutil
+
+from gyan.common.i18n import _
+import gyan.conf
+
+LOG = logging.getLogger(__name__)
+
+CONF = gyan.conf.CONF
+
+try:
+    CONF.import_opt('fatal_exception_format_errors',
+                    'oslo_versionedobjects.exception')
+except cfg.NoSuchOptError as e:
+    # Note:work around for gyan run against master branch
+    # in devstack gate job, as gyan not branched yet
+    # versionobjects kilo/master different version can
+    # cause issue here. As it changed import group. So
+    # add here before branch to prevent gate failure.
+    # Bug: #1447873
+    CONF.import_opt('fatal_exception_format_errors',
+                    'oslo_versionedobjects.exception',
+                    group='oslo_versionedobjects')
+
+
+def wrap_exception(notifier=None, event_type=None):
+    """This decorator wraps a method to catch any exceptions.
+
+    It logs the exception as well as optionally sending
+    it to the notification system.
+    """
+    def inner(f):
+        def wrapped(self, context, *args, **kwargs):
+            # Don't store self or context in the payload, it now seems to
+            # contain confidential information.
+            try:
+                return f(self, context, *args, **kwargs)
+            except Exception as e:
+                with excutils.save_and_reraise_exception():
+                    if notifier:
+                        call_dict = inspect.getcallargs(f, self, context,
+                                                        *args, **kwargs)
+                        payload = dict(exception=e,
+                                       private=dict(args=call_dict)
+                                       )
+
+                        temp_type = event_type
+                        if not temp_type:
+                            # If f has multiple decorators, they must use
+                            # functools.wraps to ensure the name is
+                            # propagated.
+                            temp_type = f.__name__
+
+                        notifier.error(context, temp_type, payload)
+
+        return functools.wraps(f)(wrapped)
+    return inner
+
+
+OBFUSCATED_MSG = _('Your request could not be handled '
+                   'because of a problem in the server. '
+                   'Error Correlation id is: %s')
+
+
+def wrap_controller_exception(func, func_server_error, func_client_error):
+    """This decorator wraps controllers methods to handle exceptions:
+
+    - if an unhandled Exception or a GyanException with an error code >=500
+      is catched, raise a http 5xx ClientSideError and correlates it with a log
+      message
+
+    - if a GyanException is catched and its error code is <500, raise a http
+      4xx and logs the excp in debug mode
+
+    """
+    @functools.wraps(func)
+    def wrapped(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception as excp:
+            if isinstance(excp, GyanException):
+                http_error_code = excp.code
+            else:
+                http_error_code = 500
+
+            if http_error_code >= 500:
+                # log the error message with its associated
+                # correlation id
+                log_correlation_id = uuidutils.generate_uuid()
+                LOG.exception("%(correlation_id)s:%(excp)s",
+                              {'correlation_id': log_correlation_id,
+                               'excp': str(excp)})
+                # raise a client error with an obfuscated message
+                return func_server_error(log_correlation_id, http_error_code)
+            else:
+                # raise a client error the original message
+                LOG.debug(excp)
+                return func_client_error(excp, http_error_code)
+    return wrapped
+
+
+def convert_excp_to_err_code(excp_name):
+    """Convert Exception class name (CamelCase) to error-code (Snake-case)"""
+    words = re.findall(r'[A-Z]?[a-z]+|[A-Z]{2,}(?=[A-Z][a-z]|\d|\W|$)|\d+',
+                       excp_name)
+    return '-'.join([str.lower(word) for word in words])
+
+
+def wrap_pecan_controller_exception(func):
+    """This decorator wraps pecan controllers to handle exceptions."""
+    def _func_server_error(log_correlation_id, status_code):
+        pecan.response.status = status_code
+        return {
+            'faultcode': 'Server',
+            'status_code': status_code,
+            'title': woutil.status_reasons[status_code],
+            'description': six.text_type(OBFUSCATED_MSG % log_correlation_id),
+        }
+
+    def _func_client_error(excp, status_code):
+        pecan.response.status = status_code
+        return {
+            'faultcode': 'Client',
+            'faultstring': convert_excp_to_err_code(excp.__class__.__name__),
+            'status_code': status_code,
+            'title': six.text_type(excp),
+            'description': six.text_type(excp),
+        }
+
+    return wrap_controller_exception(func,
+                                     _func_server_error,
+                                     _func_client_error)
+
+
+def wrap_keystone_exception(func):
+    """Wrap keystone exceptions and throw gyan specific exceptions."""
+    @functools.wraps(func)
+    def wrapped(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except keystone_exceptions.AuthorizationFailure:
+            raise AuthorizationFailure(
+                client=func.__name__, message="reason: %s" % sys.exc_info()[1])
+        except keystone_exceptions.ClientException:
+            raise AuthorizationFailure(
+                client=func.__name__,
+                message="unexpected keystone client error occurred: %s"
+                        % sys.exc_info()[1])
+    return wrapped
+
+
+class GyanException(Exception):
+    """Base gyan Exception
+
+    To correctly use this class, inherit from it and define
+    a 'message' property. That message will get printf'd
+    with the keyword arguments provided to the constructor.
+
+    """
+    message = _("An unknown exception occurred.")
+    code = 500
+
+    def __init__(self, message=None, **kwargs):
+        self.kwargs = kwargs
+
+        if 'code' not in self.kwargs and hasattr(self, 'code'):
+            self.kwargs['code'] = self.code
+
+        if message:
+            self.message = message
+
+        try:
+            self.message = str(self.message) % kwargs
+        except KeyError:
+            # kwargs doesn't match a variable in the message
+            # log the issue and the kwargs
+            LOG.exception('Exception in string format operation, '
+                          'kwargs: %s', kwargs)
+            try:
+                ferr = CONF.fatal_exception_format_errors
+            except cfg.NoSuchOptError:
+                ferr = CONF.oslo_versionedobjects.fatal_exception_format_errors
+            if ferr:
+                raise
+
+        super(GyanException, self).__init__(self.message)
+
+    def __str__(self):
+        if six.PY3:
+            return self.message
+        return self.message.encode('utf-8')
+
+    def __unicode__(self):
+        return self.message
+
+    def format_message(self):
+        if self.__class__.__name__.endswith('_Remote'):
+            return self.args[0]
+        else:
+            return six.text_type(self)
+
+
+class ObjectNotFound(GyanException):
+    message = _("The %(name)s %(id)s could not be found.")
+
+
+class ObjectNotUnique(GyanException):
+    message = _("The %(name)s already exists.")
+
+
+class ObjectActionError(GyanException):
+    message = _('Object action %(action)s failed because: %(reason)s')
+
+
+class ResourceNotFound(ObjectNotFound):
+    message = _("The %(name)s resource %(id)s could not be found.")
+    code = 404
+
+
+class ResourceExists(ObjectNotUnique):
+    message = _("The %(name)s resource already exists.")
+    code = 409
+
+
+class AuthorizationFailure(GyanException):
+    message = _("%(client)s connection failed. %(message)s")
+
+
+class UnsupportedObjectError(GyanException):
+    message = _('Unsupported object type %(objtype)s')
+
+
+class IncompatibleObjectVersion(GyanException):
+    message = _('Version %(objver)s of %(objname)s is not supported')
+
+
+class OrphanedObjectError(GyanException):
+    message = _('Cannot call %(method)s on orphaned %(objtype)s object')
+
+
+class Invalid(GyanException):
+    message = _("Unacceptable parameters.")
+    code = 400
+
+
+class InvalidValue(Invalid):
+    message = _("Received value '%(value)s' is invalid for type %(type)s.")
+
+
+class ValidationError(Invalid):
+    message = "%(detail)s"
+
+
+class SchemaValidationError(ValidationError):
+    message = "%(detail)s"
+
+
+class InvalidUUID(Invalid):
+    message = _("Expected a uuid but received %(uuid)s.")
+
+
+class InvalidName(Invalid):
+    message = _("Expected a name but received %(uuid)s.")
+
+
+class InvalidDiscoveryURL(Invalid):
+    message = _("Received invalid discovery URL '%(discovery_url)s' for "
+                "discovery endpoint '%(discovery_endpoint)s'.")
+
+
+class GetDiscoveryUrlFailed(GyanException):
+    message = _("Failed to get discovery url from '%(discovery_endpoint)s'.")
+
+
+class InvalidUuidOrName(Invalid):
+    message = _("Expected a name or uuid but received %(uuid)s.")
+
+
+class InvalidIdentity(Invalid):
+    message = _("Expected an uuid or int but received %(identity)s.")
+
+
+class InvalidCsr(Invalid):
+    message = _("Received invalid csr %(csr)s.")
+
+
+class HTTPNotFound(ResourceNotFound):
+    pass
+
+
+class Conflict(GyanException):
+    message = _('Conflict.')
+    code = 409
+
+
+class ConflictOptions(Conflict):
+    message = _('Conflicting options.')
+
+
+class InvalidState(Conflict):
+    message = _("Invalid resource state.")
+
+
+# Cannot be templated as the error syntax varies.
+# msg needs to be constructed when raised.
+class InvalidParameterValue(Invalid):
+    message = _("%(err)s")
+
+
+class InvalidParamInVersion(Invalid):
+    message = _('Invalid param %(param)s because current request '
+                'version is %(req_version)s. %(param)s is only '
+                'supported from version %(min_version)s')
+
+
+class InvalidQuotaValue(Invalid):
+    message = _("Change would make usage less than 0 for the following "
+                "resources: %(unders)s")
+
+
+class InvalidQuotaMethodUsage(Invalid):
+    message = _("Wrong quota method %(method)s used on resource %(res)s")
+
+
+class PatchError(Invalid):
+    message = _("Couldn't apply patch '%(patch)s'. Reason: %(reason)s")
+
+
+class NotAuthorized(GyanException):
+    message = _("Not authorized.")
+    code = 403
+
+
+class MLModelAlreadyExists(GyanException):
+    message = _("A ML Model with %(field)s %(value)s already exists.")
+
+
+class MLModelNotFound(GyanException):
+    message = _("ML Model %(ml_model)s could not be found.")
+
+class ConfigInvalid(GyanException):
+    message = _("Invalid configuration file. %(error_msg)s")
+
+
+class PolicyNotAuthorized(NotAuthorized):
+    message = _("Policy doesn't allow %(action)s to be performed.")
+
+
+class ComputeHostNotFound(HTTPNotFound):
+    message = _("Compute host %(compute_host)s could not be found.")
+
+
+class GyanServiceNotFound(HTTPNotFound):
+    message = _("Gyan service %(binary)s on host %(host)s could not be found.")
+
+
+class ResourceProviderNotFound(HTTPNotFound):
+    message = _("Resource provider %(resource_provider)s could not be found.")
+
+
+class ResourceClassNotFound(HTTPNotFound):
+    message = _("Resource class %(resource_class)s could not be found.")
+
+
+class ComputeHostAlreadyExists(ResourceExists):
+    message = _("A compute host with %(field)s %(value)s already exists.")
+
+
+class GyanServiceAlreadyExists(ResourceExists):
+    message = _("Service %(binary)s on host %(host)s already exists.")
+
+
+class ResourceProviderAlreadyExists(ResourceExists):
+    message = _("A resource provider with %(field)s %(value)s already exists.")
+
+
+class ResourceClassAlreadyExists(ResourceExists):
+    message = _("A resource class with %(field)s %(value)s already exists.")
+
+
+class UniqueConstraintViolated(ResourceExists):
+    message = _("A resource with %(fields)s violates unique constraint.")
+
+
+class InvalidStateException(GyanException):
+    message = _("Cannot %(action)s ml model %(id)s in %(actual_state)s state")
+    code = 409
+
+
+class ServerInError(GyanException):
+    message = _('Went to status %(resource_status)s due to '
+                '"%(status_reason)s"')
+
+
+class ServerUnknownStatus(GyanException):
+    message = _('%(result)s - Unknown status %(resource_status)s due to '
+                '"%(status_reason)s"')
+
+
+class EntityNotFound(GyanException):
+    message = _("The %(entity)s (%(name)s) could not be found.")
+
+
+class CommandError(GyanException):
+    message = _("The command: %(cmd)s failed on the system, due to %(error)s")
+
+
+class NoValidHost(GyanException):
+    message = _("No valid host was found. %(reason)s")
+
+
+class NotFound(GyanException):
+    message = _("Resource could not be found.")
+    code = 404
+
+
+class SchedulerHostFilterNotFound(NotFound):
+    message = _("Scheduler Host Filter %(filter_name)s could not be found.")
+
+
+class ClassNotFound(NotFound):
+    message = _("Class %(class_name)s could not be found: %(exception)s")
+
+
+class ApiVersionsIntersect(GyanException):
+    message = _("Version of %(name)s %(min_ver)s %(max_ver)s intersects "
+                "with another versions.")
+
+
+class ConnectionFailed(GyanException):
+    message = _("Failed to connect to remote host")
+
+
+class SocketException(GyanException):
+    message = _("Socket exceptions")
+
+
+class ResourcesUnavailable(GyanException):
+    message = _("Insufficient compute resources: %(reason)s.")
+
+
+class FileNotFound(GyanException):
+    message = _("The expected file not exist")
+
+
+class FailedParseStringToJson(GyanException):
+    message = _("Failed parse string to json: %(reason)s.")
+
+
+class ServerNotUsable(GyanException):
+    message = _("gyan server not usable")
+    code = 404
+
+
+class Base64Exception(Invalid):
+    msg_fmt = _("Invalid Base 64 file data")

gyan/common/i18n.py

@@ -0,0 +1,24 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# It's based on oslo.i18n usage in OpenStack Keystone project and
+# recommendations from
+# https://docs.openstack.org/oslo.i18n/latest/user/usage.html
+
+import oslo_i18n
+
+
+_translators = oslo_i18n.TranslatorFactory(domain='gyan')
+
+# The primary translation function using the well-known name "_"
+_ = _translators.primary

gyan/common/keystone.py

@@ -0,0 +1,89 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from keystoneauth1.access import access as ka_access
+from keystoneauth1.identity import access as ka_access_plugin
+from keystoneauth1.identity import v3 as ka_v3
+from keystoneauth1 import loading as ka_loading
+from keystoneclient.v3 import client as kc_v3
+from oslo_log import log as logging
+
+from gyan.common import exception
+import gyan.conf
+from gyan.conf import keystone as ksconf
+
+
+CONF = gyan.conf.CONF
+LOG = logging.getLogger(__name__)
+
+
+class KeystoneClientV3(object):
+    """Keystone client wrapper so we can encapsulate logic in one place."""
+
+    def __init__(self, context):
+        self.context = context
+        self._client = None
+        self._session = None
+
+    @property
+    def auth_url(self):
+        # FIXME(pauloewerton): auth_url should be retrieved from keystone_auth
+        # section by default
+        url = CONF[ksconf.CFG_LEGACY_GROUP].www_authenticate_uri or \
+            CONF[ksconf.CFG_LEGACY_GROUP].auth_uri
+        return url.replace('v2.0', 'v3')
+
+    @property
+    def auth_token(self):
+        return self.session.get_token()
+
+    @property
+    def session(self):
+        if self._session:
+            return self._session
+        auth = self._get_auth()
+        session = self._get_session(auth)
+        self._session = session
+        return session
+
+    def _get_session(self, auth):
+        session = ka_loading.load_session_from_conf_options(
+            CONF, ksconf.CFG_GROUP, auth=auth)
+        return session
+
+    def _get_auth(self):
+        if self.context.auth_token_info:
+            access_info = ka_access.create(body=self.context.auth_token_info,
+                                           auth_token=self.context.auth_token)
+            auth = ka_access_plugin.AccessInfoPlugin(access_info)
+        elif self.context.auth_token:
+            auth = ka_v3.Token(auth_url=self.auth_url,
+                               token=self.context.auth_token)
+        elif self.context.is_admin:
+            auth = ka_loading.load_auth_from_conf_options(CONF,
+                                                          ksconf.CFG_GROUP)
+        else:
+            msg = ('Keystone API connection failed: no password, '
+                   'trust_id or token found.')
+            LOG.error(msg)
+            raise exception.AuthorizationFailure(client='keystone',
+                                                 message='reason %s' % msg)
+
+        return auth
+
+    @property
+    def client(self):
+        if self._client:
+            return self._client
+        client = kc_v3.Client(session=self.session)
+        self._client = client
+        return client

gyan/common/paths.py

@@ -0,0 +1,47 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+import gyan.conf
+
+CONF = gyan.conf.CONF
+
+
+def basedir_def(*args):
+    """Return an uninterpolated path relative to $pybasedir."""
+    return os.path.join('$pybasedir', *args)
+
+
+def bindir_def(*args):
+    """Return an uninterpolated path relative to $bindir."""
+    return os.path.join('$bindir', *args)
+
+
+def state_path_def(*args):
+    """Return an uninterpolated path relative to $state_path."""
+    return os.path.join('$state_path', *args)
+
+
+def basedir_rel(*args):
+    """Return a path relative to $pybasedir."""
+    return os.path.join(CONF.pybasedir, *args)
+
+
+def bindir_rel(*args):
+    """Return a path relative to $bindir."""
+    return os.path.join(CONF.bindir, *args)
+
+
+def state_path_rel(*args):
+    """Return a path relative to $state_path."""
+    return os.path.join(CONF.state_path, *args)

gyan/common/policies/__init__.py

@@ -0,0 +1,24 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import itertools
+
+from gyan.common.policies import host
+from gyan.common.policies import base
+from gyan.common.policies import ml_model
+
+def list_rules():
+    return itertools.chain(
+        base.list_rules(),
+        host.list_rules(),
+        ml_model.list_rules()
+    )

gyan/common/policies/base.py

@@ -0,0 +1,41 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_policy import policy
+
+ROLE_ADMIN = 'role:admin'
+RULE_ADMIN_OR_OWNER = 'is_admin:True or project_id:%(project_id)s'
+RULE_ADMIN_API = 'rule:context_is_admin'
+RULE_DENY_EVERYBODY = 'rule:deny_everybody'
+
+rules = [
+    policy.RuleDefault(
+        name='context_is_admin',
+        check_str=ROLE_ADMIN
+    ),
+    policy.RuleDefault(
+        name='admin_or_owner',
+        check_str=RULE_ADMIN_OR_OWNER
+    ),
+    policy.RuleDefault(
+        name='admin_api',
+        check_str=RULE_ADMIN_API
+    ),
+    policy.RuleDefault(
+        name="deny_everybody",
+        check_str="!",
+        description="Default rule for deny everybody."),
+]
+
+
+def list_rules():
+    return rules

gyan/common/policies/host.py

@@ -0,0 +1,46 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_policy import policy
+
+from gyan.common.policies import base
+
+HOST = 'host:%s'
+
+rules = [
+    policy.DocumentedRuleDefault(
+        name=HOST % 'get_all',
+        check_str=base.RULE_ADMIN_API,
+        description='List all compute hosts.',
+        operations=[
+            {
+                'path': '/v1/hosts',
+                'method': 'GET'
+            }
+        ]
+    ),
+    policy.DocumentedRuleDefault(
+        name=HOST % 'get',
+        check_str=base.RULE_ADMIN_API,
+        description='Show the details of a specific compute host.',
+        operations=[
+            {
+                'path': '/v1/hosts/{host_ident}',
+                'method': 'GET'
+            }
+        ]
+    )
+]
+
+
+def list_rules():
+    return rules

gyan/common/policies/ml_model.py

@@ -0,0 +1,123 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_policy import policy
+
+from gyan.common.policies import base
+
+ML_MODEL = 'ml_model:%s'
+
+rules = [
+    policy.DocumentedRuleDefault(
+        name=ML_MODEL % 'create',
+        check_str=base.RULE_ADMIN_OR_OWNER,
+        description='Create a new ML Model.',
+        operations=[
+            {
+                'path': '/v1/ml_models',
+                'method': 'POST'
+            }
+        ]
+    ),
+    policy.DocumentedRuleDefault(
+        name=ML_MODEL % 'delete',
+        check_str=base.RULE_ADMIN_OR_OWNER,
+        description='Delete a ML Model.',
+        operations=[
+            {
+                'path': '/v1/ml_models/{ml_model_ident}',
+                'method': 'DELETE'
+            }
+        ]
+    ),
+    policy.DocumentedRuleDefault(
+        name=ML_MODEL % 'delete_all_projects',
+        check_str=base.RULE_ADMIN_API,
+        description='Delete a ml models from all projects.',
+        operations=[
+            {
+                'path': '/v1/ml_models/{ml_model_ident}',
+                'method': 'DELETE'
+            }
+        ]
+    ),
+    policy.DocumentedRuleDefault(
+        name=ML_MODEL % 'delete_force',
+        check_str=base.RULE_ADMIN_API,
+        description='Forcibly delete a ML model.',
+        operations=[
+            {
+                'path': '/v1/ml_models/{ml_model_ident}',
+                'method': 'DELETE'
+            }
+        ]
+    ),
+    policy.DocumentedRuleDefault(
+        name=ML_MODEL % 'get_one',
+        check_str=base.RULE_ADMIN_OR_OWNER,
+        description='Retrieve the details of a specific ml model.',
+        operations=[
+            {
+                'path': '/v1/ml_models/{ml_model_ident}',
+                'method': 'GET'
+            }
+        ]
+    ),
+    policy.DocumentedRuleDefault(
+        name=ML_MODEL % 'get_all',
+        check_str=base.RULE_ADMIN_OR_OWNER,
+        description='Retrieve the details of all ml models.',
+        operations=[
+            {
+                'path': '/v1/ml_models',
+                'method': 'GET'
+            }
+        ]
+    ),
+    policy.DocumentedRuleDefault(
+        name=ML_MODEL % 'get_all_all_projects',
+        check_str=base.RULE_ADMIN_API,
+        description='Retrieve the details of all ml models across projects.',
+        operations=[
+            {
+                'path': '/v1/ml_models',
+                'method': 'GET'
+            }
+        ]
+    ),
+    policy.DocumentedRuleDefault(
+        name=ML_MODEL % 'update',
+        check_str=base.RULE_ADMIN_OR_OWNER,
+        description='Update a ML Model.',
+        operations=[
+            {
+                'path': '/v1/ml_models/{ml_model_ident}',
+                'method': 'PATCH'
+            }
+        ]
+    ),
+    policy.DocumentedRuleDefault(
+        name=ML_MODEL % 'upload',
+        check_str=base.RULE_ADMIN_OR_OWNER,
+        description='Upload the trained ML Model',
+        operations=[
+            {
+                'path': '/v1/ml_models/{ml_model_ident}/upload',
+                'method': 'POST'
+            }
+        ]
+    ),
+]
+
+
+def list_rules():
+    return rules

gyan/common/policy.py

@@ -0,0 +1,155 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Policy Engine For Gyan."""
+
+from oslo_log import log as logging
+from oslo_policy import policy
+from oslo_utils import excutils
+
+from gyan.common import exception
+from gyan.common import policies
+import gyan.conf
+
+_ENFORCER = None
+CONF = gyan.conf.CONF
+LOG = logging.getLogger(__name__)
+
+
+# we can get a policy enforcer by this init.
+# oslo policy support change policy rule dynamically.
+# at present, policy.enforce will reload the policy rules when it checks
+# the policy files have been touched.
+def init(policy_file=None, rules=None,
+         default_rule=None, use_conf=True, overwrite=True):
+    """Init an Enforcer class.
+
+        :param policy_file: Custom policy file to use, if none is
+                            specified, ``conf.policy_file`` will be
+                            used.
+        :param rules: Default dictionary / Rules to use. It will be
+                      considered just in the first instantiation. If
+                      :meth:`load_rules` with ``force_reload=True``,
+                      :meth:`clear` or :meth:`set_rules` with
+                      ``overwrite=True`` is called this will be overwritten.
+        :param default_rule: Default rule to use, conf.default_rule will
+                             be used if none is specified.
+        :param use_conf: Whether to load rules from cache or config file.
+        :param overwrite: Whether to overwrite existing rules when reload rules
+                          from config file.
+    """
+    global _ENFORCER
+    if not _ENFORCER:
+        # https://docs.openstack.org/oslo.policy/latest/user/usage.html
+        _ENFORCER = policy.Enforcer(CONF,
+                                    policy_file=policy_file,
+                                    rules=rules,
+                                    default_rule=default_rule,
+                                    use_conf=use_conf,
+                                    overwrite=overwrite)
+        register_rules(_ENFORCER)
+    return _ENFORCER
+
+
+def register_rules(enforcer):
+    enforcer.register_defaults(policies.list_rules())
+
+
+def enforce(context, rule=None, target=None,
+            do_raise=True, exc=None, *args, **kwargs):
+
+    """Checks authorization of a rule against the target and credentials.
+
+        :param dict context: As much information about the user performing the
+                             action as possible.
+        :param rule: The rule to evaluate.
+        :param dict target: As much information about the object being operated
+                            on as possible.
+        :param do_raise: Whether to raise an exception or not if check
+                         fails.
+        :param exc: Class of the exception to raise if the check fails.
+                    Any remaining arguments passed to :meth:`enforce` (both
+                    positional and keyword arguments) will be passed to
+                    the exception class. If not specified,
+                    :class:`PolicyNotAuthorized` will be used.
+
+        :return: ``False`` if the policy does not allow the action and `exc` is
+                 not provided; otherwise, returns a value that evaluates to
+                 ``True``.  Note: for rules using the "case" expression, this
+                 ``True`` value will be the specified string from the
+                 expression.
+    """
+    enforcer = init()
+    credentials = context.to_policy_values()
+    if not exc:
+        exc = exception.PolicyNotAuthorized
+    if target is None:
+        target = {'project_id': context.project_id,
+                  'user_id': context.user_id}
+    return enforcer.enforce(rule, target, credentials,
+                            do_raise=do_raise, exc=exc, *args, **kwargs)
+
+
+def authorize(context, action, target, do_raise=True, exc=None,
+              might_not_exist=False):
+    """Verifies that the action is valid on the target in this context.
+
+       :param context: gyan context
+       :param action: string representing the action to be checked
+           this should be colon separated for clarity.
+           i.e. ``network:attach_external_network``
+       :param target: dictionary representing the object of the action
+           for object creation this should be a dictionary representing the
+           location of the object e.g. ``{'project_id': context.project_id}``
+       :param do_raise: if True (the default), raises PolicyNotAuthorized;
+           if False, returns False
+       :param exc: Class of the exception to raise if the check fails.
+            Any remaining arguments passed to :meth:`authorize` (both
+            positional and keyword arguments) will be passed to
+            the exception class. If not specified,
+            :class:`PolicyNotAuthorized` will be used.
+       :param might_not_exist: If True the policy check is skipped (and the
+            function returns True) if the specified policy does not exist.
+            Defaults to false.
+
+       :raises gyan.common.exception.PolicyNotAuthorized: if verification fails
+           and do_raise is True. Or if 'exc' is specified it will raise an
+           exception of that type.
+
+       :return: returns a non-False value (not necessarily "True") if
+           authorized, and the exact value False if not authorized and
+           do_raise is False.
+    """
+    credentials = context.to_policy_values()
+    if not exc:
+        exc = exception.PolicyNotAuthorized
+    if might_not_exist and not (_ENFORCER.rules and action in _ENFORCER.rules):
+        return True
+    try:
+        result = _ENFORCER.enforce(action, target, credentials,
+                                   do_raise=do_raise, exc=exc, action=action)
+    except Exception:
+        with excutils.save_and_reraise_exception():
+            LOG.debug('Policy check for %(action)s failed with credentials '
+                      '%(credentials)s',
+                      {'action': action, 'credentials': credentials})
+    return result
+
+
+def check_is_admin(context):
+    """Whether or not user is admin according to policy setting.
+
+    """
+    init()
+    target = {}
+    credentials = context.to_policy_values()
+    return _ENFORCER.enforce('context_is_admin', target, credentials)

gyan/common/privileged.py

@@ -0,0 +1,22 @@
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+
+from oslo_privsep import capabilities as c
+from oslo_privsep import priv_context
+
+
+default = priv_context.PrivContext(
+    'gyan.common',
+    cfg_section='privsep',
+    pypath=__name__ + '.default',
+    capabilities=[c.CAP_SYS_ADMIN],
+)

gyan/common/profiler.py

@@ -0,0 +1,101 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+###
+# This code is taken from nova. Goal is minimal modification.
+###
+
+from oslo_log import log as logging
+from oslo_utils import importutils
+import webob.dec
+
+from gyan.common import context
+import gyan.conf
+
+profiler = importutils.try_import("osprofiler.profiler")
+profiler_initializer = importutils.try_import("osprofiler.initializer")
+profiler_web = importutils.try_import("osprofiler.web")
+
+
+CONF = gyan.conf.CONF
+
+LOG = logging.getLogger(__name__)
+
+
+class WsgiMiddleware(object):
+
+    def __init__(self, application, **kwargs):
+        self.application = application
+
+    @classmethod
+    def factory(cls, global_conf, **local_conf):
+        if profiler_web:
+            return profiler_web.WsgiMiddleware.factory(global_conf,
+                                                       **local_conf)
+
+        def filter_(app):
+            return cls(app, **local_conf)
+
+        return filter_
+
+    @webob.dec.wsgify
+    def __call__(self, request):
+        return request.get_response(self.application)
+
+
+def setup(binary, host):
+    if profiler_initializer and CONF.profiler.enabled:
+        profiler_initializer.init_from_conf(
+            conf=CONF,
+            context=context.get_admin_context().to_dict(),
+            project="gyan",
+            service=binary,
+            host=host)
+        LOG.info('OSProfiler is enabled.')
+
+
+def trace_cls(name, **kwargs):
+    """Wrap the OSprofiler trace_cls.
+
+    Wrap the OSprofiler trace_cls decorator so that it will not try to
+    patch the class unless OSprofiler is present.
+
+    :param name: The name of action. For example, wsgi, rpc, db, ...
+    :param kwargs: Any other keyword args used by profiler.trace_cls
+    """
+
+    def decorator(cls):
+        if profiler and 'profiler' in CONF:
+            trace_decorator = profiler.trace_cls(name, kwargs)
+            return trace_decorator(cls)
+        return cls
+
+    return decorator
+
+
+def trace(name, **kwargs):
+    """Wrap the OSprofiler trace.
+
+    Wrap the OSprofiler trace decorator so that it will not try to
+    patch the functions unless OSprofiler is present.
+
+    :param name: The name of action. For example, wsgi, rpc, db, ...
+    :param kwargs: Any other keyword args used by profiler.trace
+    """
+
+    def decorator(f):
+        if profiler and 'profiler' in CONF:
+            trace_decorator = profiler.trace(name, kwargs)
+            return trace_decorator(f)
+        return f
+
+    return decorator

gyan/common/rpc.py

@@ -0,0 +1,120 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+__all__ = [
+    'init',
+    'set_defaults',
+    'add_extra_exmods',
+    'clear_extra_exmods',
+    'get_allowed_exmods',
+    'RequestContextSerializer',
+    'get_client',
+]
+
+import oslo_messaging as messaging
+from oslo_serialization import jsonutils as json
+from oslo_utils import importutils
+
+from gyan.common import context as gyan_context
+from gyan.common import exception
+
+profiler = importutils.try_import("osprofiler.profiler")
+
+TRANSPORT = None
+ALLOWED_EXMODS = [
+    exception.__name__,
+]
+EXTRA_EXMODS = []
+
+
+def init(conf):
+    global TRANSPORT
+    exmods = get_allowed_exmods()
+    TRANSPORT = messaging.get_rpc_transport(
+        conf, allowed_remote_exmods=exmods)
+
+
+def set_defaults(control_exchange):
+    messaging.set_transport_defaults(control_exchange)
+
+
+def add_extra_exmods(*args):
+    EXTRA_EXMODS.extend(args)
+
+
+def clear_extra_exmods():
+    del EXTRA_EXMODS[:]
+
+
+def get_allowed_exmods():
+    return ALLOWED_EXMODS + EXTRA_EXMODS
+
+
+class JsonPayloadSerializer(messaging.NoOpSerializer):
+    @staticmethod
+    def serialize_entity(context, entity):
+        return json.to_primitive(entity, convert_instances=True)
+
+
+class RequestContextSerializer(messaging.Serializer):
+
+    def __init__(self, base):
+        self._base = base
+
+    def serialize_entity(self, context, entity):
+        if not self._base:
+            return entity
+        return self._base.serialize_entity(context, entity)
+
+    def deserialize_entity(self, context, entity):
+        if not self._base:
+            return entity
+        return self._base.deserialize_entity(context, entity)
+
+    def serialize_context(self, context):
+        return context.to_dict()
+
+    def deserialize_context(self, context):
+        return gyan_context.RequestContext.from_dict(context)
+
+
+class ProfilerRequestContextSerializer(RequestContextSerializer):
+    def serialize_context(self, context):
+        _context = super(ProfilerRequestContextSerializer,
+                         self).serialize_context(context)
+
+        prof = profiler.get()
+        if prof:
+            trace_info = {
+                "hmac_key": prof.hmac_key,
+                "base_id": prof.get_base_id(),
+                "parent_id": prof.get_id()
+            }
+            _context.update({"trace_info": trace_info})
+
+        return _context
+
+    def deserialize_context(self, context):
+        trace_info = context.pop("trace_info", None)
+        if trace_info:
+            profiler.init(**trace_info)
+
+        return super(ProfilerRequestContextSerializer,
+                     self).deserialize_context(context)
+
+
+def get_client(target, serializer=None, timeout=None):
+    assert TRANSPORT is not None
+    return messaging.RPCClient(TRANSPORT,
+                               target,
+                               serializer=serializer,
+                               timeout=timeout)

gyan/common/rpc_service.py

@@ -0,0 +1,105 @@
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+"""Common RPC service and API tools for Gyan."""
+
+import oslo_messaging as messaging
+from oslo_messaging.rpc import dispatcher
+from oslo_service import service
+from oslo_utils import importutils
+
+from gyan.common import context
+from gyan.common import profiler
+from gyan.common import rpc
+import gyan.conf
+from gyan.objects import base as objects_base
+from gyan.servicegroup import gyan_service_periodic as servicegroup
+
+osprofiler = importutils.try_import("osprofiler.profiler")
+
+CONF = gyan.conf.CONF
+
+
+def _init_serializer():
+    serializer = rpc.RequestContextSerializer(
+        objects_base.GyanObjectSerializer())
+    if osprofiler:
+        serializer = rpc.ProfilerRequestContextSerializer(serializer)
+    else:
+        serializer = rpc.RequestContextSerializer(serializer)
+    return serializer
+
+
+class Service(service.Service):
+
+    def __init__(self, topic, server, endpoints, binary):
+        super(Service, self).__init__()
+        serializer = _init_serializer()
+        transport = messaging.get_rpc_transport(CONF)
+        access_policy = dispatcher.DefaultRPCAccessPolicy
+        # TODO(asalkeld) add support for version='x.y'
+        target = messaging.Target(topic=topic, server=server)
+        self.endpoints = endpoints
+        self._server = messaging.get_rpc_server(transport, target, endpoints,
+                                                executor='eventlet',
+                                                serializer=serializer,
+                                                access_policy=access_policy)
+        self.binary = binary
+        profiler.setup(binary, CONF.compute.host)
+
+    def start(self):
+        servicegroup.setup(CONF, self.binary, self.tg)
+        for endpoint in self.endpoints:
+            if hasattr(endpoint, 'init_containers'):
+                endpoint.init_containers(
+                    context.get_admin_context(all_projects=True))
+            self.tg.add_dynamic_timer(
+                endpoint.run_periodic_tasks,
+                periodic_interval_max=CONF.periodic_interval_max,
+                context=context.get_admin_context(all_projects=True)
+            )
+        self._server.start()
+
+    def stop(self):
+        if self._server:
+            self._server.stop()
+            self._server.wait()
+        super(Service, self).stop()
+
+    @classmethod
+    def create(cls, topic, server, handlers, binary):
+        service_obj = cls(topic, server, handlers, binary)
+        return service_obj
+
+
+class API(object):
+    def __init__(self, context=None, topic=None, server=None,
+                 timeout=None):
+        serializer = _init_serializer()
+        self._context = context
+        if topic is None:
+            topic = ''
+        target = messaging.Target(topic=topic, server=server)
+        self._client = rpc.get_client(target,
+                                      serializer=serializer,
+                                      timeout=timeout)
+
+    def _call(self, server, method, *args, **kwargs):
+        cctxt = self._client.prepare(server=server)
+        return cctxt.call(self._context, method, *args, **kwargs)
+
+    def _cast(self, server, method, *args, **kwargs):
+        cctxt = self._client.prepare(server=server)
+        return cctxt.cast(self._context, method, *args, **kwargs)
+
+    def echo(self, message):
+        self._cast('echo', message=message)

gyan/common/service.py

@@ -0,0 +1,92 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+from oslo_concurrency import processutils
+from oslo_log import log
+from oslo_service import service
+from oslo_service import wsgi
+
+from gyan.api import app
+from gyan.common import config
+from gyan.common import exception
+from gyan.common.i18n import _
+import gyan.conf
+
+CONF = gyan.conf.CONF
+
+
+def prepare_service(argv=None):
+    if argv is None:
+        argv = []
+    log.register_options(CONF)
+    config.parse_args(argv)
+    config.set_config_defaults()
+    log.setup(CONF, 'gyan')
+    # TODO(yuanying): Uncomment after objects are implemented
+    # objects.register_all()
+
+
+def process_launcher():
+    return service.ProcessLauncher(CONF, restart_method='mutate')
+
+
+class WSGIService(service.ServiceBase):
+    """Provides ability to launch Gyan API from wsgi app."""
+
+    def __init__(self, name, use_ssl=False):
+        """Initialize, but do not start the WSGI server.
+
+        :param name: The name of the WSGI server given to the loader.
+        :param use_ssl: Wraps the socket in an SSL context if True.
+        :returns: None
+        """
+        self.name = name
+        self.app = app.load_app()
+        self.workers = (CONF.api.workers or processutils.get_worker_count())
+        if self.workers and self.workers < 1:
+            raise exception.ConfigInvalid(
+                _("api_workers value of %d is invalid, "
+                  "must be greater than 0.") % self.workers)
+
+        self.server = wsgi.Server(CONF, name, self.app,
+                                  host=CONF.api.host_ip,
+                                  port=CONF.api.port,
+                                  use_ssl=use_ssl)
+
+    def start(self):
+        """Start serving this service using loaded configuration.
+
+        :returns: None
+        """
+        self.server.start()
+
+    def stop(self):
+        """Stop serving this API.
+
+        :returns: None
+        """
+        self.server.stop()
+
+    def wait(self):
+        """Wait for the service to stop serving this API.
+
+        :returns: None
+        """
+        self.server.wait()
+
+    def reset(self):
+        """Reset server greenpool size to default.
+
+        :returns: None
+        """
+        self.server.reset()

gyan/common/short_id.py

@@ -0,0 +1,63 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Utilities for creating short ID strings based on a random UUID.
+The IDs each comprise 12 (lower-case) alphanumeric characters.
+"""
+
+import base64
+from oslo_utils import uuidutils
+import uuid
+
+import six
+
+from gyan.common.i18n import _
+
+
+def _to_byte_string(value, num_bits):
+    """Convert an integer to a big-endian string of bytes with padding.
+
+    Padding is added at the end (i.e. after the least-significant bit) if
+    required.
+    """
+
+    shifts = six.moves.xrange(num_bits - 8, -8, -8)
+    byte_at = lambda off: (value >> off if off >= 0 else value << -off) & 0xff
+    return ''.join(six.int2byte(byte_at(offset)) for offset in shifts)
+
+
+def get_id(source_uuid):
+    """Derive a short (12 character) id from a random UUID.
+
+    The supplied UUID must be a version 4 UUID object.
+    """
+
+    if isinstance(source_uuid, six.string_types):
+        source_uuid = uuid.UUID(source_uuid)
+    if source_uuid.version != 4:
+        raise ValueError(_('Invalid UUID version (%d)') % source_uuid.version)
+
+    # The "time" field of a v4 UUID contains 60 random bits
+    # (see RFC4122, Section 4.4)
+    random_bytes = _to_byte_string(source_uuid.time, 60)
+    # The first 12 bytes (= 60 bits) of base32-encoded output is our data
+    encoded = base64.b32encode(six.b(random_bytes))[:12]
+
+    if six.PY3:
+        return encoded.lower().decode('utf-8')
+    else:
+        return encoded.lower()
+
+
+def generate_id():
+    """Generate a short (12 character), random id."""
+    return uuidutils.generate_uuid()

gyan/common/singleton.py

@@ -0,0 +1,25 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_concurrency import lockutils
+
+
+class Singleton(type):
+    _instances = {}
+    _semaphores = lockutils.Semaphores()
+
+    def __call__(cls, *args, **kwargs):
+        with lockutils.lock('singleton_lock', semaphores=cls._semaphores):
+            if cls not in cls._instances:
+                cls._instances[cls] = super(
+                    Singleton, cls).__call__(*args, **kwargs)
+        return cls._instances[cls]

gyan/common/utils.py

@@ -0,0 +1,255 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# It's based on oslo.i18n usage in OpenStack Keystone project and
+# recommendations from
+# https://docs.openstack.org/oslo.i18n/latest/user/usage.html
+
+"""Utilities and helper functions."""
+import base64
+import binascii
+import eventlet
+import functools
+import inspect
+import json
+import mimetypes
+
+from oslo_concurrency import processutils
+from oslo_context import context as common_context
+from oslo_log import log as logging
+from oslo_utils import excutils
+from oslo_utils import strutils
+import pecan
+import six
+
+from gyan.api import utils as api_utils
+from gyan.common import consts
+from gyan.common import exception
+from gyan.common.i18n import _
+from gyan.common import privileged
+import gyan.conf
+from gyan import objects
+
+CONF = gyan.conf.CONF
+LOG = logging.getLogger(__name__)
+
+VALID_STATES = {
+    'deploy': [consts.CREATED, consts.UNDEPLOYED],
+    'undeploy': [consts.DEPLOYED]
+}
+def safe_rstrip(value, chars=None):
+    """Removes trailing characters from a string if that does not make it empty
+
+    :param value: A string value that will be stripped.
+    :param chars: Characters to remove.
+    :return: Stripped value.
+
+    """
+    if not isinstance(value, six.string_types):
+        LOG.warning(
+            "Failed to remove trailing character. Returning original object. "
+            "Supplied object is not a string: %s.", value)
+        return value
+
+    return value.rstrip(chars) or value
+
+
+def _do_allow_certain_content_types(func, content_types_list):
+    # Allows you to bypass pecan's content-type restrictions
+    cfg = pecan.util._cfg(func)
+    cfg.setdefault('content_types', {})
+    cfg['content_types'].update((value, '')
+                                for value in content_types_list)
+    return func
+
+
+def allow_certain_content_types(*content_types_list):
+    def _wrapper(func):
+        return _do_allow_certain_content_types(func, content_types_list)
+    return _wrapper
+
+
+def allow_all_content_types(f):
+    return _do_allow_certain_content_types(f, mimetypes.types_map.values())
+
+
+def spawn_n(func, *args, **kwargs):
+    """Passthrough method for eventlet.spawn_n.
+
+    This utility exists so that it can be stubbed for testing without
+    interfering with the service spawns.
+
+    It will also grab the context from the threadlocal store and add it to
+    the store on the new thread.  This allows for continuity in logging the
+    context when using this method to spawn a new thread.
+    """
+    _context = common_context.get_current()
+
+    @functools.wraps(func)
+    def context_wrapper(*args, **kwargs):
+        # NOTE: If update_store is not called after spawn_n it won't be
+        # available for the logger to pull from threadlocal storage.
+        if _context is not None:
+            _context.update_store()
+        func(*args, **kwargs)
+
+    eventlet.spawn_n(context_wrapper, *args, **kwargs)
+
+
+def translate_exception(function):
+    """Wraps a method to catch exceptions.
+
+    If the exception is not an instance of GyanException,
+    translate it into one.
+    """
+
+    @functools.wraps(function)
+    def decorated_function(self, context, *args, **kwargs):
+        try:
+            return function(self, context, *args, **kwargs)
+        except Exception as e:
+            if not isinstance(e, exception.GyanException):
+                LOG.exception("Unexpected error: %s", six.text_type(e))
+                e = exception.GyanException("Unexpected error: %s"
+                                           % six.text_type(e))
+                raise e
+            raise
+
+    return decorated_function
+
+
+def custom_execute(*cmd, **kwargs):
+    try:
+        return processutils.execute(*cmd, **kwargs)
+    except processutils.ProcessExecutionError as e:
+        sanitized_cmd = strutils.mask_password(' '.join(cmd))
+        raise exception.CommandError(cmd=sanitized_cmd,
+                                     error=six.text_type(e))
+
+
+def is_all_projects(search_opts):
+    all_projects = search_opts.get('all_projects')
+    if all_projects:
+        try:
+            all_projects = strutils.bool_from_string(all_projects, True)
+        except ValueError:
+            bools = ', '.join(strutils.TRUE_STRINGS + strutils.FALSE_STRINGS)
+            raise exception.InvalidValue(_('Valid all_projects values are: %s')
+                                         % bools)
+    else:
+        all_projects = False
+    return all_projects
+
+
+def get_ml_model(ml_model_ident):
+    ml_model = api_utils.get_resource('ML_Model', ml_model_ident)
+    if not ml_model:
+        pecan.abort(404, ('Not found; the ml model you requested '
+                          'does not exist.'))
+
+    return ml_model
+
+def validate_ml_model_state(ml_model, action):
+    if ml_model.status not in VALID_STATES[action]:
+        raise exception.InvalidStateException(
+            id=ml_model.uuid,
+            action=action,
+            actual_state=ml_model.status)
+
+
+def get_wrapped_function(function):
+    """Get the method at the bottom of a stack of decorators."""
+    if not hasattr(function, '__closure__') or not function.__closure__:
+        return function
+
+    def _get_wrapped_function(function):
+        if not hasattr(function, '__closure__') or not function.__closure__:
+            return None
+
+        for closure in function.__closure__:
+            func = closure.cell_contents
+
+            deeper_func = _get_wrapped_function(func)
+            if deeper_func:
+                return deeper_func
+            elif hasattr(closure.cell_contents, '__call__'):
+                return closure.cell_contents
+
+        return function
+
+    return _get_wrapped_function(function)
+
+
+def wrap_ml_model_event(prefix):
+    """Warps a method to log the event taken on the ml_model, and result.
+
+    This decorator wraps a method to log the start and result of an event, as
+    part of an action taken on a ml_model.
+    """
+    def helper(function):
+
+        @functools.wraps(function)
+        def decorated_function(self, context, *args, **kwargs):
+            wrapped_func = get_wrapped_function(function)
+            keyed_args = inspect.getcallargs(wrapped_func, self, context,
+                                             *args, **kwargs)
+            ml_model_uuid = keyed_args['ml_model'].uuid
+
+            event_name = '{0}_{1}'.format(prefix, function.__name__)
+            with EventReporter(context, event_name, ml_model_uuid):
+                return function(self, context, *args, **kwargs)
+        return decorated_function
+    return helper
+
+
+def wrap_exception():
+    def helper(function):
+
+        @functools.wraps(function)
+        def decorated_function(self, context, ml_model, *args, **kwargs):
+            try:
+                return function(self, context, ml_model, *args, **kwargs)
+            except exception.DockerError as e:
+                with excutils.save_and_reraise_exception(reraise=False):
+                    LOG.error("Error occurred while calling Docker API: %s",
+                              six.text_type(e))
+            except Exception as e:
+                with excutils.save_and_reraise_exception(reraise=False):
+                    LOG.exception("Unexpected exception: %s", six.text_type(e))
+        return decorated_function
+    return helper
+
+
+def is_close(x, y, rel_tol=1e-06, abs_tol=0.0):
+    return abs(x - y) <= max(rel_tol * max(abs(x), abs(y)), abs_tol)
+
+
+def is_less_than(x, y):
+    if isinstance(x, int) and isinstance(y, int):
+        return x < y
+    if isinstance(x, float) or isinstance(y, float):
+        return False if (x - y) >= 0 or is_close(x, y) else True
+
+
+def encode_file_data(data):
+    if six.PY3 and isinstance(data, str):
+        data = data.encode('utf-8')
+    return base64.b64encode(data).decode('utf-8')
+
+
+def decode_file_data(data):
+    # Py3 raises binascii.Error instead of TypeError as in Py27
+    try:
+        return base64.b64decode(data)
+    except (TypeError, binascii.Error):
+        raise exception.Base64Exception()

gyan/common/yamlutils.py

@@ -0,0 +1,33 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import yaml
+
+
+def load(s):
+    try:
+        yml_dict = yaml.safe_load(s)
+    except yaml.YAMLError as exc:
+        msg = 'An error occurred during YAML parsing.'
+        if hasattr(exc, 'problem_mark'):
+            msg += ' Error position: (%s:%s)' % (exc.problem_mark.line + 1,
+                                                 exc.problem_mark.column + 1)
+        raise ValueError(msg)
+    if not isinstance(yml_dict, dict) and not isinstance(yml_dict, list):
+        raise ValueError('The source is not a YAML mapping or list.')
+    if isinstance(yml_dict, dict) and len(yml_dict) < 1:
+        raise ValueError('Could not find any element in your YAML mapping.')
+    return yml_dict
+
+
+def dump(s):
+    return yaml.safe_dump(s)

gyan/compute/api.py

@@ -0,0 +1,63 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Handles all requests relating to compute resources (e.g. ml_models,
+and compute hosts on which they run)."""
+
+from oslo_log import log as logging
+
+from gyan.common import consts
+from gyan.common import exception
+from gyan.common.i18n import _
+from gyan.common import profiler
+from gyan.compute import rpcapi
+import gyan.conf
+from gyan import objects
+
+
+CONF = gyan.conf.CONF
+LOG = logging.getLogger(__name__)
+
+
+@profiler.trace_cls("rpc")
+class API(object):
+    """API for interacting with the compute manager."""
+
+    def __init__(self, context):
+        self.rpcapi = rpcapi.API(context=context)
+        super(API, self).__init__()
+
+    def ml_model_create(self, context, new_ml_model, extra_spec):
+        try:
+            host_state = self._schedule_ml_model(context, ml_model,
+                                                  extra_spec)
+        except exception.NoValidHost:
+            new_ml_model.status = consts.ERROR
+            new_ml_model.status_reason = _(
+                "There are not enough hosts available.")
+            new_ml_model.save(context)
+            return
+        except Exception:
+            new_ml_model.status = consts.ERROR
+            new_ml_model.status_reason = _("Unexpected exception occurred.")
+            new_ml_model.save(context)
+            raise
+
+        self.rpcapi.ml_model_create(context, host_state['host'],
+                                     new_ml_model)
+
+    def ml_model_delete(self, context, ml_model, *args):
+        self._record_action_start(context, ml_model, ml_model_actions.DELETE)
+        return self.rpcapi.ml_model_delete(context, ml_model, *args)
+
+    def ml_model_show(self, context, ml_model):
+        return self.rpcapi.ml_model_show(context, ml_model)

gyan/compute/compute_host_tracker.py

@@ -0,0 +1,67 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import collections
+import copy
+
+from oslo_log import log as logging
+
+from gyan.common import exception
+from gyan.common import utils
+from gyan import objects
+from gyan.objects import base as obj_base
+
+LOG = logging.getLogger(__name__)
+COMPUTE_RESOURCE_SEMAPHORE = "compute_resources"
+
+
+class ComputeHostTracker(object):
+    def __init__(self, host, ml_model_driver):
+        self.host = host
+        self.ml_model_driver = ml_model_driver
+        self.compute_host = None
+        self.tracked_ml_models = {}
+        self.old_resources = collections.defaultdict(objects.ComputeHost)
+        
+        
+    def update_available_resources(self, context):
+        # Check if the compute_host is already registered
+        host = self._get_compute_host(context)
+        if not host:
+            # If not, register it and pass the object to the driver
+            host = objects.ComputeHost(context)
+            host.hostname = self.host
+            host.type = self.ml_model_driver.__class__.__name__
+            host.status = "AVAILABLE"
+            host.create(context)
+            LOG.info('Host created for :%(host)s', {'host': self.host})
+        self.ml_model_driver.get_available_resources(host)
+        self.compute_host = host
+        return host
+
+    def _get_compute_host(self, context):
+        """Returns compute host for the host"""
+        try:
+            return objects.ComputeHost.get_by_name(context, self.host)
+        except exception.ComputeHostNotFound:
+            LOG.warning("No compute host record for: %(host)s",
+                        {'host': self.host})
+
+    
+    def _set_ml_model_host(self, context, ml_model):
+        """Tag the ml_model as belonging to this host.
+
+        This should be done while the COMPUTE_RESOURCES_SEMAPHORE is held so
+        the resource claim will not be lost if the audit process starts.
+        """
+        ml_model.host = self.host
+        ml_model.save(context)

gyan/compute/manager.py

@@ -0,0 +1,121 @@
+#    Copyright 2016 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import itertools
+
+import six
+import time
+
+from oslo_log import log as logging
+from oslo_service import periodic_task
+from oslo_utils import excutils
+from oslo_utils import timeutils
+from oslo_utils import uuidutils
+
+from gyan.common import consts
+from gyan.common import context
+from gyan.common import exception
+from gyan.common.i18n import _
+from gyan.common import utils
+from gyan.common.utils import translate_exception
+from gyan.common.utils import wrap_ml_model_event
+from gyan.common.utils import wrap_exception
+from gyan.compute import compute_host_tracker
+import gyan.conf
+from gyan.ml_model import driver
+from gyan import objects
+
+CONF = gyan.conf.CONF
+LOG = logging.getLogger(__name__)
+
+
+class Manager(periodic_task.PeriodicTasks):
+    """Manages the running ml_models."""
+
+    def __init__(self, ml_model_driver=None):
+        super(Manager, self).__init__(CONF)
+        self.driver = driver.load_ml_model_driver(ml_model_driver)
+        self.host = CONF.compute.host
+        self._resource_tracker = None
+
+    def ml_model_create(self, context, limits, requested_networks,
+                         requested_volumes, ml_model, run, pci_requests=None):
+        @utils.synchronized(ml_model.uuid)
+        def do_ml_model_create():
+            created_ml_model = self._do_ml_model_create(
+                context, ml_model, requested_networks, requested_volumes,
+                pci_requests, limits)
+            if run:
+                self._do_ml_model_start(context, created_ml_model)
+
+        utils.spawn_n(do_ml_model_create)
+
+    @wrap_ml_model_event(prefix='compute')
+    def _do_ml_model_create(self, context, ml_model, requested_networks,
+                             requested_volumes, pci_requests=None,
+                             limits=None):
+        LOG.debug('Creating ml_model: %s', ml_model.uuid)
+
+        try:
+            rt = self._get_resource_tracker()
+            # As sriov port also need to claim, we need claim pci port before
+            # create sandbox.
+            with rt.ml_model_claim(context, ml_model, pci_requests, limits):
+                sandbox = None
+                if self.use_sandbox:
+                    sandbox = self._create_sandbox(context, ml_model,
+                                                   requested_networks)
+
+                created_ml_model = self._do_ml_model_create_base(
+                    context, ml_model, requested_networks, requested_volumes,
+                    sandbox, limits)
+                return created_ml_model
+        except exception.ResourcesUnavailable as e:
+            with excutils.save_and_reraise_exception():
+                LOG.exception("ML Model resource claim failed: %s",
+                              six.text_type(e))
+                self._fail_ml_model(context, ml_model, six.text_type(e),
+                                     unset_host=True)
+
+    @wrap_ml_model_event(prefix='compute')
+    def _do_ml_model_start(self, context, ml_model):
+        pass
+
+    @translate_exception
+    def ml_model_delete(self, context, ml_model, force=False):
+        pass
+
+    @translate_exception
+    def ml_model_show(self, context, ml_model):
+        pass
+
+    @translate_exception
+    def ml_model_start(self, context, ml_model):
+        pass
+
+    @translate_exception
+    def ml_model_update(self, context, ml_model, patch):
+        pass
+
+    @periodic_task.periodic_task(run_immediately=True)
+    def inventory_host(self, context):
+        rt = self._get_resource_tracker()
+        rt.update_available_resources(context)
+
+    def _get_resource_tracker(self):
+        if not self._resource_tracker:
+            rt = compute_host_tracker.ComputeHostTracker(self.host,
+                                                         self.driver)
+            self._resource_tracker = rt
+        return self._resource_tracker

gyan/compute/rpcapi.py

@@ -0,0 +1,68 @@
+#    Copyright 2016 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import functools
+
+from gyan.api import servicegroup
+from gyan.common import exception
+from gyan.common import profiler
+from gyan.common import rpc_service
+import gyan.conf
+from gyan import objects
+
+
+def check_ml_model_host(func):
+    """Verify the state of ML Model host"""
+    @functools.wraps(func)
+    def wrap(self, context, ml_model, *args, **kwargs):
+        return func(self, context, ml_model, *args, **kwargs)
+    return wrap
+
+
+@profiler.trace_cls("rpc")
+class API(rpc_service.API):
+    """Client side of the ml_model compute rpc API.
+
+    API version history:
+
+        * 1.0 - Initial version.
+    """
+
+    def __init__(self, transport=None, context=None, topic=None):
+        if topic is None:
+            gyan.conf.CONF.import_opt(
+                'topic', 'gyan.conf.compute', group='compute')
+
+        super(API, self).__init__(
+            context, gyan.conf.CONF.compute.topic, transport)
+
+    def ml_model_create(self, context, host, ml_model):
+        self._cast(host, 'ml_model_create', 
+                   ml_model=ml_model)
+
+    @check_ml_model_host
+    def ml_model_delete(self, context, ml_model, force):
+        return self._cast(ml_model.host, 'ml_model_delete',
+                          ml_model=ml_model, force=force)
+
+    @check_ml_model_host
+    def ml_model_show(self, context, ml_model):
+        return self._call(ml_model.host, 'ml_model_show',
+                          ml_model=ml_model)
+
+
+    @check_ml_model_host
+    def ml_model_update(self, context, ml_model, patch):
+        return self._call(ml_model.host, 'ml_model_update',
+                          ml_model=ml_model, patch=patch)

gyan/conf/__init__.py

@@ -0,0 +1,41 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+
+from gyan.conf import api
+from gyan.conf import compute
+from gyan.conf import ml_model_driver
+from gyan.conf import database
+from gyan.conf import keystone
+from gyan.conf import path
+from gyan.conf import profiler
+from gyan.conf import scheduler
+from gyan.conf import services
+from gyan.conf import ssl
+from gyan.conf import utils
+from gyan.conf import gyan_client
+
+CONF = cfg.CONF
+
+api.register_opts(CONF)
+compute.register_opts(CONF)
+ml_model_driver.register_opts(CONF)
+database.register_opts(CONF)
+keystone.register_opts(CONF)
+path.register_opts(CONF)
+scheduler.register_opts(CONF)
+services.register_opts(CONF)
+gyan_client.register_opts(CONF)
+ssl.register_opts(CONF)
+profiler.register_opts(CONF)
+utils.register_opts(CONF)

gyan/conf/api.py

@@ -0,0 +1,67 @@
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+from oslo_config import cfg
+
+
+api_service_opts = [
+    cfg.PortOpt('port',
+                default=8517,
+                help='The port for the gyan API server.'),
+    cfg.StrOpt('host',
+                default="localhost",
+                help='The port for the gyan API server.'),
+    cfg.IPOpt('host_ip',
+              default='127.0.0.1',
+              help="The listen IP for the gyan API server. "
+                   "The default is ``$my_ip``, "
+                   "the IP address of this host."),
+    cfg.BoolOpt('enable_ssl_api',
+                default=False,
+                help="Enable the integrated stand-alone API to service "
+                     "requests via HTTPS instead of HTTP. If there is a "
+                     "front-end service performing HTTPS offloading from "
+                     "the service, this option should be False; note, you "
+                     "will want to change public API endpoint to represent "
+                     "SSL termination URL with 'public_endpoint' option."),
+    cfg.IntOpt('workers',
+               help="Number of workers for gyan-api service. "
+                    "The default will be the number of CPUs available."),
+    cfg.IntOpt('max_limit',
+               default=1000,
+               help='The maximum number of items returned in a single '
+                    'response from a collection resource.'),
+    cfg.StrOpt('api_paste_config',
+               default="api-paste.ini",
+               help="Configuration file for WSGI definition of API."),
+    cfg.BoolOpt('enable_image_validation',
+                default=True,
+                help="Enable image validation.")
+]
+
+
+api_group = cfg.OptGroup(name='api',
+                         title='Options for the gyan-api service')
+
+
+ALL_OPTS = (api_service_opts)
+
+
+def register_opts(conf):
+    conf.register_group(api_group)
+    conf.register_opts(ALL_OPTS, api_group)
+
+
+def list_opts():
+    return {
+        api_group: ALL_OPTS
+    }

gyan/conf/compute.py

@@ -0,0 +1,40 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+
+
+service_opts = [
+    cfg.StrOpt(
+        'topic',
+        default='gyan-compute',
+        help='The queue to add compute tasks to.'),
+    cfg.StrOpt(
+        'host',
+        default='localhost',
+        help='hostname'),
+
+]
+
+opt_group = cfg.OptGroup(
+    name='compute', title='Options for the gyan-compute service')
+
+ALL_OPTS = (service_opts)
+
+
+def register_opts(conf):
+    conf.register_group(opt_group)
+    conf.register_opts(ALL_OPTS, opt_group)
+
+
+def list_opts():
+    return {opt_group: ALL_OPTS}

gyan/conf/database.py

@@ -0,0 +1,32 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+
+
+sql_opts = [
+    cfg.StrOpt('mysql_engine',
+               default='InnoDB',
+               help='MySQL engine to use.')
+]
+
+
+
+DEFAULT_OPTS = (sql_opts)
+
+
+def register_opts(conf):
+    conf.register_opts(sql_opts, 'database')
+
+
+def list_opts():
+    return {"DEFAULT": DEFAULT_OPTS}

gyan/conf/gyan_client.py

@@ -0,0 +1,51 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+
+
+gyan_group = cfg.OptGroup(name='gyan_client',
+                         title='Options for the Gyan client')
+
+gyan_client_opts = [
+    cfg.StrOpt('region_name',
+               help='Region in Identity service catalog to use for '
+                    'communication with the OpenStack service.'),
+    cfg.StrOpt('endpoint_type',
+               default='publicURL',
+               help='Type of endpoint in Identity service catalog to use '
+                    'for communication with the OpenStack service.')]
+
+
+common_security_opts = [
+    cfg.StrOpt('ca_file',
+               help='Optional CA cert file to use in SSL connections.'),
+    cfg.StrOpt('cert_file',
+               help='Optional PEM-formatted certificate chain file.'),
+    cfg.StrOpt('key_file',
+               help='Optional PEM-formatted file that contains the '
+                    'private key.'),
+    cfg.BoolOpt('insecure',
+                default=False,
+                help="If set, then the server's certificate will not "
+                     "be verified.")]
+
+ALL_OPTS = (gyan_client_opts + common_security_opts)
+
+
+def register_opts(conf):
+    conf.register_group(gyan_group)
+    conf.register_opts(gyan_client_opts, group=gyan_group)
+
+
+def list_opts():
+    return {gyan_group: ALL_OPTS}

gyan/conf/keystone.py

@@ -0,0 +1,35 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy
+# of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from keystoneauth1 import loading as ka_loading
+from oslo_config import cfg
+
+CFG_GROUP = 'keystone_auth'
+CFG_LEGACY_GROUP = 'keystone_authtoken'
+
+keystone_auth_group = cfg.OptGroup(name=CFG_GROUP,
+                                   title='Options for Keystone in Gyan')
+
+
+def register_opts(conf):
+    conf.import_group(CFG_LEGACY_GROUP, 'keystonemiddleware.auth_token')
+    ka_loading.register_auth_conf_options(conf, CFG_GROUP)
+    ka_loading.register_session_conf_options(conf, CFG_GROUP)
+    conf.set_default('auth_type', default='password', group=CFG_GROUP)
+
+
+def list_opts():
+    keystone_auth_opts = (ka_loading.get_auth_common_conf_options() +
+                          ka_loading.get_auth_plugin_conf_options('password'))
+    return {
+        keystone_auth_group: keystone_auth_opts
+    }

gyan/conf/ml_model_driver.py

@@ -0,0 +1,47 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+
+driver_opts = [
+    cfg.StrOpt('ml_model_driver',
+               default='gyan.ml_model.tensorflow.driver.TensorflowDriver',
+               help="""Defines which driver to use for controlling ml_model.
+Possible values:
+
+* ``ml_model.driver.TensorflowDriver``
+
+Services which consume this:
+
+* ``gyan-compute``
+
+Interdependencies to other options:
+
+* None
+"""),
+    cfg.IntOpt('default_sleep_time', default=1,
+               help='Time to sleep (in seconds) during waiting for an event.'),
+    cfg.IntOpt('default_timeout', default=60 * 10,
+               help='Maximum time (in seconds) to wait for an event.')
+]
+
+
+ALL_OPTS = (driver_opts)
+
+
+def register_opts(conf):
+    conf.register_opts(ALL_OPTS)
+
+
+def list_opts():
+    return {"DEFAULT": ALL_OPTS}

gyan/conf/opts.py

@@ -0,0 +1,76 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+This is the single point of entry to generate the sample configuration
+file for Gyan. It collects all the necessary info from the other modules
+in this package. It is assumed that:
+
+* every other module in this package has a 'list_opts' function which
+  return a dict where
+  * the keys are strings which are the group names
+  * the value of each key is a list of config options for that group
+* the gyan.conf package doesn't have further packages with config options
+* this module is only used in the context of sample file generation
+"""
+
+import collections
+import importlib
+import os
+import pkgutil
+
+LIST_OPTS_FUNC_NAME = "list_opts"
+
+
+def _tupleize(dct):
+    """Take the dict of options and convert to the 2-tuple format."""
+    return [(key, val) for key, val in dct.items()]
+
+
+def list_opts():
+    opts = collections.defaultdict(list)
+    module_names = _list_module_names()
+    imported_modules = _import_modules(module_names)
+    _append_config_options(imported_modules, opts)
+    return _tupleize(opts)
+
+
+def _list_module_names():
+    module_names = []
+    package_path = os.path.dirname(os.path.abspath(__file__))
+    for _, modname, ispkg in pkgutil.iter_modules(path=[package_path]):
+        if modname == "opts" or ispkg:
+            continue
+        else:
+            module_names.append(modname)
+    return module_names
+
+
+def _import_modules(module_names):
+    imported_modules = []
+    for modname in module_names:
+        mod = importlib.import_module("gyan.conf." + modname)
+        if not hasattr(mod, LIST_OPTS_FUNC_NAME):
+            msg = "The module 'gyan.conf.%s' should have a '%s' "\
+                  "function which returns the config options." % \
+                  (modname, LIST_OPTS_FUNC_NAME)
+            raise AttributeError(msg)
+        else:
+            imported_modules.append(mod)
+    return imported_modules
+
+
+def _append_config_options(imported_modules, config_options):
+    for mod in imported_modules:
+        configs = mod.list_opts()
+        for key, val in configs.items():
+            config_options[key].extend(val)

gyan/conf/path.py

@@ -0,0 +1,42 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+
+from oslo_config import cfg
+
+
+path_opts = [
+    cfg.StrOpt('pybasedir',
+               default=os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                                    '../')),
+               help='Directory where the gyan python module is installed.'),
+    cfg.StrOpt('bindir',
+               default='$pybasedir/bin',
+               help='Directory where gyan binaries are installed.'),
+    cfg.StrOpt('state_path',
+               default='$pybasedir',
+               help="Top-level directory for maintaining gyan's state."),
+]
+
+
+def state_path_def(*args):
+    """Return an uninterpolated path relative to $state_path."""
+    return os.path.join('$state_path', *args)
+
+
+def register_opts(conf):
+    conf.register_opts(path_opts)
+
+
+def list_opts():
+    return {"DEFAULT": path_opts}

gyan/conf/profiler.py

@@ -0,0 +1,29 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy
+# of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_utils import importutils
+
+
+profiler_opts = importutils.try_import('osprofiler.opts')
+
+
+def register_opts(conf):
+    if profiler_opts:
+        profiler_opts.set_defaults(conf)
+
+
+def list_opts():
+    if not profiler_opts:
+        return {}
+    return {
+        profiler_opts._profiler_opt_group: profiler_opts._PROFILER_OPTS
+    }

gyan/conf/scheduler.py

@@ -0,0 +1,104 @@
+# Copyright 2015 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+
+
+scheduler_group = cfg.OptGroup(name="scheduler",
+                               title="Scheduler configuration")
+
+scheduler_opts = [
+    cfg.StrOpt("driver",
+               default="filter_scheduler",
+               choices=("chance_scheduler", "fake_scheduler",
+                        "filter_scheduler"),
+               help="""
+The class of the driver used by the scheduler.
+
+The options are chosen from the entry points under the namespace
+'gyan.scheduler.driver' in 'setup.cfg'.
+
+Possible values:
+
+* A string, where the string corresponds to the class name of a scheduler
+  driver. There are a number of options available:
+** 'chance_scheduler', which simply picks a host at random
+** A custom scheduler driver. In this case, you will be responsible for
+   creating and maintaining the entry point in your 'setup.cfg' file
+"""),
+    cfg.MultiStrOpt("available_filters",
+                    default=["gyan.scheduler.filters.all_filters"],
+                    help="""
+Filters that the scheduler can use.
+
+An unordered list of the filter classes the gyan scheduler may apply.  Only the
+filters specified in the 'scheduler_enabled_filters' option will be used, but
+any filter appearing in that option must also be included in this list.
+
+By default, this is set to all filters that are included with gyan.
+
+This option is only used by the FilterScheduler and its subclasses; if you use
+a different scheduler, this option has no effect.
+
+Possible values:
+
+* A list of zero or more strings, where each string corresponds to the name of
+  a filter that may be used for selecting a host
+
+Related options:
+
+* scheduler_enabled_filters
+"""),
+    cfg.ListOpt("enabled_filters",
+                default=[
+                    "AvailabilityZoneFilter",
+                    "CPUFilter",
+                    "RamFilter",
+                    "ComputeFilter",
+                    "DiskFilter",
+                    ],
+                help="""
+Filters that the scheduler will use.
+
+An ordered list of filter class names that will be used for filtering
+hosts. Ignore the word 'default' in the name of this option: these filters will
+*always* be applied, and they will be applied in the order they are listed so
+place your most restrictive filters first to make the filtering process more
+efficient.
+
+This option is only used by the FilterScheduler and its subclasses; if you use
+a different scheduler, this option has no effect.
+
+Possible values:
+
+* A list of zero or more strings, where each string corresponds to the name of
+  a filter to be used for selecting a host
+
+Related options:
+
+* All of the filters in this option *must* be present in the
+  'scheduler_available_filters' option, or a SchedulerHostFilterNotFound
+  exception will be raised.
+"""),
+]
+
+
+def register_opts(conf):
+    conf.register_group(scheduler_group)
+    conf.register_opts(scheduler_opts, group=scheduler_group)
+
+
+def list_opts():
+    return {scheduler_group: scheduler_opts}

gyan/conf/services.py

@@ -0,0 +1,36 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+
+
+periodic_opts = [
+    cfg.IntOpt('periodic_interval_max',
+               default=60,
+               help='Max interval size between periodic tasks execution in '
+                    'seconds.'),
+    cfg.IntOpt('service_down_time',
+               default=180,
+               help='Max interval size between periodic tasks execution in '
+                    'seconds.')
+]
+
+
+ALL_OPTS = (periodic_opts)
+
+
+def register_opts(conf):
+    conf.register_opts(ALL_OPTS)
+
+
+def list_opts():
+    return {"DEFAULT": ALL_OPTS}

gyan/conf/ssl.py

@@ -0,0 +1,27 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+from oslo_service import sslutils
+
+
+def register_opts(conf):
+    sslutils.register_opts(conf)
+
+
+def list_opts():
+    group_name, ssl_opts = sslutils.list_opts()[0]
+    ssl_group = cfg.OptGroup(name=group_name,
+                             title='Options for the ssl')
+    return {
+        ssl_group: ssl_opts
+    }

gyan/conf/utils.py

@@ -0,0 +1,31 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy
+# of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+
+
+utils_opts = [
+    cfg.StrOpt('rootwrap_config',
+               default="/etc/gyan/rootwrap.conf",
+               help='Path to the rootwrap configuration file to use for '
+                    'running commands as root.'),
+]
+
+
+def register_opts(conf):
+    conf.register_opts(utils_opts)
+
+
+def list_opts():
+    return {
+        "DEFAULT": utils_opts
+    }

gyan/db/__init__.py

@@ -0,0 +1,21 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_db import options
+
+from gyan.common import paths
+import gyan.conf
+
+_DEFAULT_SQL_CONNECTION = 'sqlite:///' + paths.state_path_def('gyan.sqlite')
+
+options.set_defaults(gyan.conf.CONF)
+options.set_defaults(gyan.conf.CONF, _DEFAULT_SQL_CONNECTION)

gyan/db/api.py

@@ -0,0 +1,197 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""
+Base API for Database
+"""
+
+from oslo_db import api as db_api
+
+from gyan.common import profiler
+import gyan.conf
+
+"""Add the database backend mapping here"""
+
+CONF = gyan.conf.CONF
+_BACKEND_MAPPING = {'sqlalchemy': 'gyan.db.sqlalchemy.api'}
+IMPL = db_api.DBAPI.from_config(CONF,
+                                backend_mapping=_BACKEND_MAPPING,
+                                lazy=True)
+
+
+@profiler.trace("db")
+def _get_dbdriver_instance():
+    """Return a DB API instance."""
+    return IMPL
+
+
+@profiler.trace("db")
+def list_ml_models(context, filters=None, limit=None, marker=None,
+                    sort_key=None, sort_dir=None):
+    """List matching ML Models.
+
+    Return a list of the specified columns for all ml models that match
+    the specified filters.
+
+    :param context: The security context
+    :param filters: Filters to apply. Defaults to None.
+    :param limit: Maximum number of ml_models to return.
+    :param marker: the last item of the previous page; we return the next
+                   result set.
+    :param sort_key: Attribute by which results should be sorted.
+    :param sort_dir: Direction in which results should be sorted.
+                     (asc, desc)
+    :returns: A list of tuples of the specified columns.
+    """
+    return _get_dbdriver_instance().list_ml_models(
+        context, filters, limit, marker, sort_key, sort_dir)
+
+
+@profiler.trace("db")
+def create_ml_model(context, values):
+    """Create a new ML Model.
+
+    :param context: The security context
+    :param values: A dict containing several items used to identify
+                   and track the ML Model
+    :returns: A ML Model.
+    """
+    return _get_dbdriver_instance().create_ml_model(context, values)
+
+
+@profiler.trace("db")
+def get_ml_model_by_uuid(context, ml_model_uuid):
+    """Return a ML Model.
+
+    :param context: The security context
+    :param ml_model_uuid: The uuid of a ml model.
+    :returns: A ML Model.
+    """
+    return _get_dbdriver_instance().get_ml_model_by_uuid(
+        context, ml_model_uuid)
+
+
+@profiler.trace("db")
+def get_ml_model_by_name(context, ml_model_name):
+    """Return a ML Model.
+
+    :param context: The security context
+    :param ml_model_name: The name of a ML Model.
+    :returns: A ML Model.
+    """
+    return _get_dbdriver_instance().get_ml_model_by_name(
+        context, ml_model_name)
+
+
+@profiler.trace("db")
+def destroy_ml_model(context, ml_model_id):
+    """Destroy a ml model and all associated interfaces.
+
+    :param context: Request context
+    :param ml_model_id: The id or uuid of a ml model.
+    """
+    return _get_dbdriver_instance().destroy_ml_model(context, ml_model_id)
+
+
+@profiler.trace("db")
+def update_ml_model(context, ml_model_id, values):
+    """Update properties of a ml model.
+
+    :param context: Request context
+    :param ml_model_id: The id or uuid of a ml model.
+    :param values: The properties to be updated
+    :returns: A ML Model.
+    :raises: MLModelNotFound
+    """
+    return _get_dbdriver_instance().update_ml_model(
+        context, ml_model_id, values)
+
+
+@profiler.trace("db")
+def list_compute_hosts(context, filters=None, limit=None, marker=None,
+                       sort_key=None, sort_dir=None):
+    """List matching compute hosts.
+
+    Return a list of the specified columns for all compute hosts that match
+    the specified filters.
+
+    :param context: The security context
+    :param filters: Filters to apply. Defaults to None.
+    :param limit: Maximum number of compute nodes to return.
+    :param marker: the last item of the previous page; we return the next
+                   result set.
+    :param sort_key: Attribute by which results should be sorted.
+    :param sort_dir: Direction in which results should be sorted.
+                     (asc, desc)
+    :returns: A list of tuples of the specified columns.
+    """
+    return _get_dbdriver_instance().list_compute_hosts(
+        context, filters, limit, marker, sort_key, sort_dir)
+
+
+@profiler.trace("db")
+def create_compute_host(context, values):
+    """Create a new compute host.
+
+    :param context: The security context
+    :param values: A dict containing several items used to identify
+                   and track the compute node, and several dicts which are
+                   passed into the Drivers when managing this compute host.
+    :returns: A compute host.
+    """
+    return _get_dbdriver_instance().create_compute_host(context, values)
+
+
+@profiler.trace("db")
+def get_compute_host(context, host_uuid):
+    """Return a compute host.
+
+    :param context: The security context
+    :param node_uuid: The uuid of a compute node.
+    :returns: A compute node.
+    """
+    return _get_dbdriver_instance().get_compute_host(context, host_uuid)
+
+
+@profiler.trace("db")
+def get_compute_host_by_hostname(context, hostname):
+    """Return a compute node.
+
+    :param context: The security context
+    :param hostname: The hostname of a compute node.
+    :returns: A compute node.
+    """
+    return _get_dbdriver_instance().get_compute_host_by_hostname(
+        context, hostname)
+
+
+@profiler.trace("db")
+def destroy_compute_host(context, host_uuid):
+    """Destroy a compute node and all associated interfaces.
+
+    :param context: Request context
+    :param node_uuid: The uuid of a compute node.
+    """
+    return _get_dbdriver_instance().destroy_compute_host(context, host_uuid)
+
+
+@profiler.trace("db")
+def update_compute_host(context, host_uuid, values):
+    """Update properties of a compute node.
+
+    :param context: Request context
+    :param node_uuid: The uuid of a compute node.
+    :param values: The properties to be updated
+    :returns: A compute node.
+    :raises: ComputeNodeNotFound
+    """
+    return _get_dbdriver_instance().update_compute_host(
+        context, host_uuid, values)

gyan/db/migration.py

@@ -0,0 +1,47 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Database setup and migration commands."""
+
+from stevedore import driver
+
+import gyan.conf
+
+_IMPL = None
+
+
+def get_backend():
+    global _IMPL
+    if not _IMPL:
+        gyan.conf.CONF.import_opt('backend',
+                                 'oslo_db.options', group='database')
+        _IMPL = driver.DriverManager("gyan.database.migration_backend",
+                                     gyan.conf.CONF.database.backend).driver
+    return _IMPL
+
+
+def upgrade(version=None):
+    """Migrate the database to `version` or the most recent version."""
+    return get_backend().upgrade(version)
+
+
+def version():
+    return get_backend().version()
+
+
+def stamp(version):
+    return get_backend().stamp(version)
+
+
+def revision(message, autogenerate):
+    return get_backend().revision(message, autogenerate)

gyan/db/sqlalchemy/alembic.ini

@@ -0,0 +1,68 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = %(here)s/alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat alembic/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+#sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

gyan/db/sqlalchemy/alembic/README

@@ -0,0 +1,11 @@
+Please see https://alembic.readthedocs.org/en/latest/index.html for general documentation
+
+To create alembic migrations use:
+$ gyan-db-manage revision --message "description of revision" --autogenerate
+
+Stamp db with most recent migration version, without actually running migrations
+$ gyan-db-manage stamp head
+
+Upgrade can be performed by:
+$ gyan-db-manage upgrade
+$ gyan-db-manage upgrade head

gyan/db/sqlalchemy/alembic/env.py

@@ -0,0 +1,58 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from __future__ import with_statement
+from alembic import context
+from logging.config import fileConfig
+
+from gyan.db.sqlalchemy import api as sqla_api
+from gyan.db.sqlalchemy import models
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = models.Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    engine = sqla_api.get_engine()
+    with engine.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            render_as_batch=True
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+run_migrations_online()

gyan/db/sqlalchemy/alembic/script.py.mako

@@ -0,0 +1,20 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}

gyan/db/sqlalchemy/alembic/versions/cebd81b206ca_add_model_and_host_tables.py

@@ -0,0 +1,48 @@
+"""Add Model and Host tables
+
+Revision ID: cebd81b206ca
+Create Date: 2018-10-09 09:57:20.823110
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'cebd81b206ca'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        'compute_host',
+        sa.Column('id', sa.String(length=255), nullable=False),
+        sa.Column('created_at', sa.DateTime(), nullable=True),
+        sa.Column('updated_at', sa.DateTime(), nullable=True),
+        sa.Column('hostname', sa.String(length=255), nullable=True),
+        sa.Column('status', sa.String(length=255), nullable=True),
+        sa.Column('type', sa.String(length=255), nullable=True),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table(
+        'ml_model',
+        sa.Column('id', sa.String(length=255), nullable=False),
+        sa.Column('created_at', sa.DateTime(), nullable=True),
+        sa.Column('updated_at', sa.DateTime(), nullable=True),
+        sa.Column('host_id', sa.String(length=255), nullable=True),
+        sa.Column('name', sa.String(length=255), nullable=True),
+        sa.Column('project_id', sa.String(length=255), nullable=True),
+        sa.Column('user_id', sa.String(length=255), nullable=True),
+        sa.Column('status', sa.String(length=255), nullable=True),
+        sa.Column('status_reason', sa.Text, nullable=True),
+        sa.Column('url', sa.Text, nullable=True),
+        sa.Column('hints', sa.Text, nullable=True),
+        sa.Column('deployed', sa.Boolean(), nullable=True),
+        sa.PrimaryKeyConstraint('id'),
+        sa.ForeignKeyConstraint(['host_id'], ['compute_host.id'])
+    )
+
+    # ### end Alembic commands ###

gyan/db/sqlalchemy/api.py

@@ -0,0 +1,306 @@
+# Copyright 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""SQLAlchemy storage backend."""
+
+from oslo_db import exception as db_exc
+from oslo_db.sqlalchemy import session as db_session
+from oslo_db.sqlalchemy import utils as db_utils
+from oslo_utils import importutils
+from oslo_utils import strutils
+from oslo_utils import timeutils
+from oslo_utils import uuidutils
+import sqlalchemy as sa
+from sqlalchemy.orm import contains_eager
+from sqlalchemy.orm.exc import MultipleResultsFound
+from sqlalchemy.orm.exc import NoResultFound
+from sqlalchemy.sql.expression import desc
+from sqlalchemy.sql import func
+
+from gyan.common import consts
+from gyan.common import exception
+from gyan.common.i18n import _
+import gyan.conf
+from gyan.db.sqlalchemy import models
+
+profiler_sqlalchemy = importutils.try_import('osprofiler.sqlalchemy')
+
+CONF = gyan.conf.CONF
+
+_FACADE = None
+
+
+def _create_facade_lazily():
+    global _FACADE
+    if _FACADE is None:
+        _FACADE = db_session.enginefacade.get_legacy_facade()
+        if profiler_sqlalchemy:
+            if CONF.profiler.enabled and CONF.profiler.trace_sqlalchemy:
+                profiler_sqlalchemy.add_tracing(sa, _FACADE.get_engine(), "db")
+    return _FACADE
+
+
+def get_engine():
+    facade = _create_facade_lazily()
+    return facade.get_engine()
+
+
+def get_session(**kwargs):
+    facade = _create_facade_lazily()
+    return facade.get_session(**kwargs)
+
+
+def get_backend():
+    """The backend is this module itself."""
+    return Connection()
+
+
+def model_query(model, *args, **kwargs):
+    """Query helper for simpler session usage.
+
+    :param session: if present, the session to use
+    """
+
+    session = kwargs.get('session') or get_session()
+    query = session.query(model, *args)
+    return query
+
+
+def add_identity_filter(query, value):
+    """Adds an identity filter to a query.
+
+    Filters results by ID, if supplied value is a valid integer.
+    Otherwise attempts to filter results by UUID.
+
+    :param query: Initial query to add filter to.
+    :param value: Value for filtering results by.
+    :return: Modified query.
+    """
+    if strutils.is_int_like(value):
+        return query.filter_by(id=value)
+    elif uuidutils.is_uuid_like(value):
+        return query.filter_by(uuid=value)
+    else:
+        raise exception.InvalidIdentity(identity=value)
+
+
+def _paginate_query(model, limit=None, marker=None, sort_key=None,
+                    sort_dir=None, query=None, default_sort_key='id'):
+    if not query:
+        query = model_query(model)
+    sort_keys = [default_sort_key]
+    if sort_key and sort_key not in sort_keys:
+        sort_keys.insert(0, sort_key)
+    try:
+        query = db_utils.paginate_query(query, model, limit, sort_keys,
+                                        marker=marker, sort_dir=sort_dir)
+    except db_exc.InvalidSortKey:
+        raise exception.InvalidParameterValue(
+            _('The sort_key value "%(key)s" is an invalid field for sorting')
+            % {'key': sort_key})
+    return query.all()
+
+
+class Connection(object):
+    """SqlAlchemy connection."""
+
+    def __init__(self):
+        pass
+
+    def _add_project_filters(self, context, query):
+        if context.is_admin and context.all_projects:
+            return query
+
+        if context.project_id:
+            query = query.filter_by(project_id=context.project_id)
+        else:
+            query = query.filter_by(user_id=context.user_id)
+
+        return query
+
+    def _add_filters(self, query, model, filters=None, filter_names=None):
+        """Generic way to add filters to a Gyan model"""
+        if not filters:
+            return query
+
+        if not filter_names:
+            filter_names = []
+
+        for name in filter_names:
+            if name in filters:
+                value = filters[name]
+                if isinstance(value, list):
+                    column = getattr(model, name)
+                    query = query.filter(column.in_(value))
+                else:
+                    column = getattr(model, name)
+                    query = query.filter(column == value)
+
+        return query
+
+    def _add_compute_hosts_filters(self, query, filters):
+        filter_names = None
+        return self._add_filters(query, models.ComputeHost, filters=filters,
+                                 filter_names=filter_names)
+
+    def list_compute_hosts(self, context, filters=None, limit=None,
+                           marker=None, sort_key=None, sort_dir=None):
+        query = model_query(models.ComputeHost)
+        query = self._add_compute_hosts_filters(query, filters)
+        return _paginate_query(models.ComputeHost, limit, marker,
+                               sort_key, sort_dir, query,
+                               default_sort_key='id')
+
+    def create_compute_host(self, context, values):
+        # ensure defaults are present for new compute hosts
+        if not values.get('id'):
+            values['id'] = uuidutils.generate_uuid()
+
+        compute_host = models.ComputeHost()
+        compute_host.update(values)
+        try:
+            compute_host.save()
+        except db_exc.DBDuplicateEntry:
+            raise exception.ComputeHostAlreadyExists(
+                field='UUID', value=values['uuid'])
+        return compute_host
+
+    def get_compute_host(self, context, host_uuid):
+        query = model_query(models.ComputeHost)
+        query = query.filter_by(id=host_uuid)
+        try:
+            return query.one()
+        except NoResultFound:
+            raise exception.ComputeHostNotFound(
+                compute_host=host_uuid)
+
+    def get_compute_host_by_hostname(self, context, hostname):
+        query = model_query(models.ComputeHost)
+        query = query.filter_by(hostname=hostname)
+        try:
+            return query.one()
+        except NoResultFound:
+            raise exception.ComputeHostNotFound(
+                compute_host=hostname)
+        except MultipleResultsFound:
+            raise exception.Conflict('Multiple compute hosts exist with same '
+                                     'hostname. Please use the uuid instead.')
+
+    def destroy_compute_host(self, context, host_uuid):
+        session = get_session()
+        with session.begin():
+            query = model_query(models.ComputeHost, session=session)
+            query = query.filter_by(uuid=host_uuid)
+            count = query.delete()
+            if count != 1:
+                raise exception.ComputeHostNotFound(
+                    compute_host=host_uuid)
+
+    def update_compute_host(self, context, host_uuid, values):
+        if 'uuid' in values:
+            msg = _("Cannot overwrite UUID for an existing ComputeHost.")
+            raise exception.InvalidParameterValue(err=msg)
+
+        return self._do_update_compute_host(host_uuid, values)
+
+    def _do_update_compute_host(self, host_uuid, values):
+        session = get_session()
+        with session.begin():
+            query = model_query(models.ComputeHost, session=session)
+            query = query.filter_by(uuid=host_uuid)
+            try:
+                ref = query.with_lockmode('update').one()
+            except NoResultFound:
+                raise exception.ComputeHostNotFound(
+                    compute_host=host_uuid)
+
+            ref.update(values)
+        return ref
+
+    def list_ml_models(self, context, filters=None, limit=None,
+                      marker=None, sort_key=None, sort_dir=None):
+        query = model_query(models.Capsule)
+        query = self._add_project_filters(context, query)
+        query = self._add_ml_models_filters(query, filters)
+        return _paginate_query(models.Capsule, limit, marker,
+                               sort_key, sort_dir, query)
+
+    def create_ml_model(self, context, values):
+        # ensure defaults are present for new ml_models
+        if not values.get('uuid'):
+            values['uuid'] = uuidutils.generate_uuid()
+        ml_model = models.ML_Model()
+        ml_model.update(values)
+        try:
+            ml_model.save()
+        except db_exc.DBDuplicateEntry:
+            raise exception.MLModelAlreadyExists(field='UUID',
+                                                 value=values['uuid'])
+        return ml_model
+
+    def get_ml_model_by_uuid(self, context, ml_model_uuid):
+        query = model_query(models.ML_Model)
+        query = self._add_project_filters(context, query)
+        query = query.filter_by(uuid=ml_model_uuid)
+        try:
+            return query.one()
+        except NoResultFound:
+            raise exception.MLModelNotFound(ml_model=ml_model_uuid)
+
+    def get_ml_model_by_name(self, context, ml_model_name):
+        query = model_query(models.ML_Model)
+        query = self._add_project_filters(context, query)
+        query = query.filter_by(meta_name=ml_model_name)
+        try:
+            return query.one()
+        except NoResultFound:
+            raise exception.MLModelNotFound(ml_model=ml_model_name)
+        except MultipleResultsFound:
+            raise exception.Conflict('Multiple ml_models exist with same '
+                                     'name. Please use the ml_model uuid '
+                                     'instead.')
+
+    def destroy_ml_model(self, context, ml_model_id):
+        session = get_session()
+        with session.begin():
+            query = model_query(models.ML_Model, session=session)
+            query = add_identity_filter(query, ml_model_id)
+            count = query.delete()
+            if count != 1:
+                raise exception.MLModelNotFound(ml_model_id)
+
+    def update_ml_model(self, context, ml_model_id, values):
+        if 'uuid' in values:
+            msg = _("Cannot overwrite UUID for an existing ML Model.")
+            raise exception.InvalidParameterValue(err=msg)
+
+        return self._do_update_ml_model_id(ml_model_id, values)
+
+    def _do_update_ml_model_id(self, ml_model_id, values):
+        session = get_session()
+        with session.begin():
+            query = model_query(models.ML_Model, session=session)
+            query = add_identity_filter(query, ml_model_id)
+            try:
+                ref = query.with_lockmode('update').one()
+            except NoResultFound:
+                raise exception.MLModelNotFound(ml_model=ml_model_id)
+
+            ref.update(values)
+        return ref
+
+    def _add_ml_models_filters(self, query, filters):
+        filter_names = ['uuid', 'project_id', 'user_id']
+        return self._add_filters(query, models.ML_Model, filters=filters,
+                                 filter_names=filter_names)

gyan/db/sqlalchemy/migration.py

@@ -0,0 +1,111 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+import alembic
+from alembic import config as alembic_config
+import alembic.migration as alembic_migration
+from oslo_db import exception as db_exc
+from oslo_db.sqlalchemy import enginefacade
+from oslo_db.sqlalchemy.migration_cli import manager
+
+from gyan.db.sqlalchemy import models
+
+import gyan.conf
+
+_MANAGER = None
+
+
+def _alembic_config():
+    path = os.path.join(os.path.dirname(__file__), 'alembic.ini')
+    config = alembic_config.Config(path)
+    return config
+
+
+def get_manager():
+    global _MANAGER
+    if not _MANAGER:
+        alembic_path = os.path.abspath(
+            os.path.join(os.path.dirname(__file__), 'alembic.ini'))
+        migrate_path = os.path.abspath(
+            os.path.join(os.path.dirname(__file__), 'alembic'))
+        migration_config = {'alembic_ini_path': alembic_path,
+                            'alembic_repo_path': migrate_path,
+                            'db_url': gyan.conf.CONF.database.connection}
+        _MANAGER = manager.MigrationManager(migration_config)
+
+    return _MANAGER
+
+
+def version(config=None, engine=None):
+    """Current database version.
+
+    :returns: Database version
+    :rtype: string
+    """
+    if engine is None:
+        engine = enginefacade.get_legacy_facade().get_engine()
+    with engine.connect() as conn:
+        context = alembic_migration.MigrationContext.configure(conn)
+        return context.get_current_revision()
+
+
+def upgrade(version):
+    """Used for upgrading database.
+
+    :param version: Desired database version
+    :type version: string
+    """
+    version = version or 'head'
+
+    get_manager().upgrade(version)
+
+
+def stamp(revision, config=None):
+    """Stamps database with provided revision.
+
+    Don't run any migrations.
+
+    :param revision: Should match one from repository or head - to stamp
+                     database with most recent revision
+    :type revision: string
+    """
+    config = config or _alembic_config()
+    return alembic.command.stamp(config, revision=revision)
+
+
+def create_schema(config=None, engine=None):
+    """Create database schema from models description.
+
+    Can be used for initial installation instead of upgrade('head').
+    """
+    if engine is None:
+        engine = enginefacade.get_legacy_facade().get_engine()
+
+    if version(engine=engine) is not None:
+        raise db_exc.DBMigrationError("DB schema is already under version"
+                                      " control. Use upgrade() instead")
+    models.Base.metadata.create_all(engine)
+    stamp('head', config=config)
+
+
+def revision(message=None, autogenerate=False):
+    """Creates template for migration.
+
+    :param message: Text that will be used for migration title
+    :type message: string
+    :param autogenerate: If True - generates diff based on current database
+                         state
+    :type autogenerate: bool
+    """
+    return get_manager().revision(message=message, autogenerate=autogenerate)