[WIP] Add hosts APIs and compute rpc APIs
Change-Id: I5893bf066fc876e795f020214080fb92b79a00a5
This commit is contained in:
bharath
parent
2f791de40e
commit
a40f556027
9
devstack/README.rst
Normal file
9
devstack/README.rst
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
====================
|
||||||
|
DevStack Integration
|
||||||
|
====================
|
||||||
|
|
||||||
|
This directory contains the files necessary to integrate gyan with devstack.
|
||||||
|
|
||||||
|
Refer the quickstart guide at
|
||||||
|
https://docs.openstack.org/gyan/latest/contributor/quickstart.html
|
||||||
|
for more information on using devstack and gyan.
|
||||||
325
devstack/lib/gyan
Normal file
325
devstack/lib/gyan
Normal file
@ -0,0 +1,325 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# lib/gyan
|
||||||
|
# Functions to control the configuration and operation of the **gyan** service
|
||||||
|
|
||||||
|
# Dependencies:
|
||||||
|
#
|
||||||
|
# - ``functions`` file
|
||||||
|
# - ``DEST``, ``DATA_DIR``, ``STACK_USER`` must be defined
|
||||||
|
# - ``SERVICE_{TENANT_NAME|PASSWORD}`` must be defined
|
||||||
|
|
||||||
|
# ``stack.sh`` calls the entry points in this order:
|
||||||
|
#
|
||||||
|
# - install_gyan
|
||||||
|
# - configure_gyan
|
||||||
|
# - create_gyan_conf
|
||||||
|
# - create_gyan_accounts
|
||||||
|
# - init_gyan
|
||||||
|
# - start_gyan
|
||||||
|
# - stop_gyan
|
||||||
|
# - cleanup_gyan
|
||||||
|
|
||||||
|
# Save trace setting
|
||||||
|
XTRACE=$(set +o | grep xtrace)
|
||||||
|
set +o xtrace
|
||||||
|
|
||||||
|
|
||||||
|
# Defaults
|
||||||
|
# --------
|
||||||
|
|
||||||
|
# Set up default directories
|
||||||
|
GYAN_REPO=${GYAN_REPO:-${GIT_BASE}/openstack/gyan.git}
|
||||||
|
GYAN_BRANCH=${GYAN_BRANCH:-master}
|
||||||
|
GYAN_DIR=$DEST/gyan
|
||||||
|
|
||||||
|
GITREPO["python-gyanclient"]=${GYANCLIENT_REPO:-${GIT_BASE}/openstack/python-gyanclient.git}
|
||||||
|
GITBRANCH["python-gyanclient"]=${GYANCLIENT_BRANCH:-master}
|
||||||
|
GITDIR["python-gyanclient"]=$DEST/python-gyanclient
|
||||||
|
|
||||||
|
GYAN_STATE_PATH=${GYAN_STATE_PATH:=$DATA_DIR/gyan}
|
||||||
|
GYAN_AUTH_CACHE_DIR=${GYAN_AUTH_CACHE_DIR:-/var/cache/gyan}
|
||||||
|
|
||||||
|
GYAN_CONF_DIR=/etc/gyan
|
||||||
|
GYAN_CONF=$GYAN_CONF_DIR/gyan.conf
|
||||||
|
GYAN_API_PASTE=$GYAN_CONF_DIR/api-paste.ini
|
||||||
|
|
||||||
|
if is_ssl_enabled_service "gyan" || is_service_enabled tls-proxy; then
|
||||||
|
GYAN_SERVICE_PROTOCOL="https"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Toggle for deploying GYAN-API under a wsgi server
|
||||||
|
GYAN_USE_UWSGI=${GYAN_USE_UWSGI:-True}
|
||||||
|
|
||||||
|
|
||||||
|
# Public facing bits
|
||||||
|
GYAN_SERVICE_HOST=${GYAN_SERVICE_HOST:-$SERVICE_HOST}
|
||||||
|
GYAN_SERVICE_PORT=${GYAN_SERVICE_PORT:-8517}
|
||||||
|
GYAN_SERVICE_PORT_INT=${GYAN_SERVICE_PORT_INT:-18517}
|
||||||
|
GYAN_SERVICE_PROTOCOL=${GYAN_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
|
||||||
|
|
||||||
|
GYAN_TRUSTEE_DOMAIN_ADMIN_PASSWORD=${GYAN_TRUSTEE_DOMAIN_ADMIN_PASSWORD:-secret}
|
||||||
|
|
||||||
|
# Support entry points installation of console scripts
|
||||||
|
if [[ -d $GYAN_DIR/bin ]]; then
|
||||||
|
GYAN_BIN_DIR=$GYAN_DIR/bin
|
||||||
|
else
|
||||||
|
GYAN_BIN_DIR=$(get_python_exec_prefix)
|
||||||
|
fi
|
||||||
|
|
||||||
|
GYAN_UWSGI=$GYAN_BIN_DIR/gyan-api-wsgi
|
||||||
|
GYAN_UWSGI_CONF=$GYAN_CONF_DIR/gyan-api-uwsgi.ini
|
||||||
|
|
||||||
|
GYAN_DB_TYPE=${GYAN_DB_TYPE:-sql}
|
||||||
|
|
||||||
|
if is_ubuntu; then
|
||||||
|
UBUNTU_RELEASE_BASE_NUM=`lsb_release -r | awk '{print $2}' | cut -d '.' -f 1`
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Functions
|
||||||
|
# ---------
|
||||||
|
|
||||||
|
# cleanup_gyan() - Remove residual data files, anything left over from previous
|
||||||
|
# runs that a clean run would need to clean up
|
||||||
|
function cleanup_gyan {
|
||||||
|
sudo rm -rf $GYAN_STATE_PATH $GYAN_AUTH_CACHE_DIR
|
||||||
|
|
||||||
|
remove_uwsgi_config "$GYAN_UWSGI_CONF" "$GYAN_UWSGI"
|
||||||
|
}
|
||||||
|
|
||||||
|
# configure_gyan() - Set config files, create data dirs, etc
|
||||||
|
function configure_gyan {
|
||||||
|
# Put config files in ``/etc/gyan`` for everyone to find
|
||||||
|
if [[ ! -d $GYAN_CONF_DIR ]]; then
|
||||||
|
sudo mkdir -p $GYAN_CONF_DIR
|
||||||
|
sudo chown $STACK_USER $GYAN_CONF_DIR
|
||||||
|
fi
|
||||||
|
|
||||||
|
configure_rootwrap gyan
|
||||||
|
|
||||||
|
# Rebuild the config file from scratch
|
||||||
|
create_gyan_conf
|
||||||
|
|
||||||
|
create_api_paste_conf
|
||||||
|
|
||||||
|
write_uwsgi_config "$GYAN_UWSGI_CONF" "$GYAN_UWSGI" "/ml-infra"
|
||||||
|
|
||||||
|
if [[ "$USE_PYTHON3" = "True" ]]; then
|
||||||
|
# Switch off glance->swift communication as swift fails under py3.x
|
||||||
|
iniset /etc/glance/glance-api.conf glance_store default_store file
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# create_gyan_accounts() - Set up common required GYAN accounts
|
||||||
|
#
|
||||||
|
# Project User Roles
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# SERVICE_PROJECT_NAME gyan service
|
||||||
|
function create_gyan_accounts {
|
||||||
|
|
||||||
|
create_service_user "gyan" "admin"
|
||||||
|
|
||||||
|
if is_service_enabled gyan-api; then
|
||||||
|
|
||||||
|
local gyan_api_url
|
||||||
|
if [[ "$GYAN_USE_UWSGI" == "True" ]]; then
|
||||||
|
gyan_api_url="$GYAN_SERVICE_PROTOCOL://$GYAN_SERVICE_HOST/ml-infra"
|
||||||
|
else
|
||||||
|
gyan_api_url="$GYAN_SERVICE_PROTOCOL://$GYAN_SERVICE_HOST:$GYAN_SERVICE_PORT"
|
||||||
|
fi
|
||||||
|
|
||||||
|
local gyan_service=$(get_or_create_service "gyan" \
|
||||||
|
"ml-infra" "ML Infra As Service")
|
||||||
|
get_or_create_endpoint $gyan_service \
|
||||||
|
"$REGION_NAME" \
|
||||||
|
"$gyan_api_url/v1" \
|
||||||
|
"$gyan_api_url/v1" \
|
||||||
|
"$gyan_api_url/v1"
|
||||||
|
fi
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
# create_gyan_conf() - Create a new gyan.conf file
|
||||||
|
function create_gyan_conf {
|
||||||
|
|
||||||
|
# (Re)create ``gyan.conf``
|
||||||
|
rm -f $GYAN_CONF
|
||||||
|
if [[ ${GYAN_DRIVER} == "tensorflow" ]]; then
|
||||||
|
iniset $GYAN_CONF DEFAULT ml_model_driver "ml_model.driver.TensorflowDriver"
|
||||||
|
fi
|
||||||
|
if [[ ${GYAN_DB_TYPE} == "sql" ]]; then
|
||||||
|
iniset $GYAN_CONF DEFAULT db_type sql
|
||||||
|
fi
|
||||||
|
iniset $GYAN_CONF DEFAULT debug "$ENABLE_DEBUG_LOG_LEVEL"
|
||||||
|
iniset $GYAN_CONF DEFAULT my_ip "$HOST_IP"
|
||||||
|
iniset $GYAN_CONF DEFAULT host "$HOST_IP"
|
||||||
|
iniset $GYAN_CONF oslo_messaging_rabbit rabbit_userid $RABBIT_USERID
|
||||||
|
iniset $GYAN_CONF oslo_messaging_rabbit rabbit_password $RABBIT_PASSWORD
|
||||||
|
iniset $GYAN_CONF oslo_messaging_rabbit rabbit_host $RABBIT_HOST
|
||||||
|
iniset $GYAN_CONF database connection `database_connection_url gyan`
|
||||||
|
iniset $GYAN_CONF api host_ip "$GYAN_SERVICE_HOST"
|
||||||
|
iniset $GYAN_CONF api port "$GYAN_SERVICE_PORT"
|
||||||
|
|
||||||
|
iniset $GYAN_CONF keystone_auth auth_type password
|
||||||
|
iniset $GYAN_CONF keystone_auth username gyan
|
||||||
|
iniset $GYAN_CONF keystone_auth password $SERVICE_PASSWORD
|
||||||
|
iniset $GYAN_CONF keystone_auth project_name $SERVICE_PROJECT_NAME
|
||||||
|
iniset $GYAN_CONF keystone_auth project_domain_id default
|
||||||
|
iniset $GYAN_CONF keystone_auth user_domain_id default
|
||||||
|
|
||||||
|
# FIXME(pauloewerton): keystone_authtoken section is deprecated. Remove it
|
||||||
|
# after deprecation period.
|
||||||
|
iniset $GYAN_CONF keystone_authtoken admin_user gyan
|
||||||
|
iniset $GYAN_CONF keystone_authtoken admin_password $SERVICE_PASSWORD
|
||||||
|
iniset $GYAN_CONF keystone_authtoken admin_tenant_name $SERVICE_PROJECT_NAME
|
||||||
|
|
||||||
|
configure_auth_token_middleware $GYAN_CONF gyan $GYAN_AUTH_CACHE_DIR
|
||||||
|
|
||||||
|
iniset $GYAN_CONF keystone_auth auth_url $KEYSTONE_AUTH_URI_V3
|
||||||
|
iniset $GYAN_CONF keystone_authtoken www_authenticate_uri $KEYSTONE_SERVICE_URI_V3
|
||||||
|
iniset $GYAN_CONF keystone_authtoken auth_url $KEYSTONE_AUTH_URI_V3
|
||||||
|
iniset $GYAN_CONF keystone_authtoken auth_version v3
|
||||||
|
|
||||||
|
|
||||||
|
if is_fedora || is_suse; then
|
||||||
|
# gyan defaults to /usr/local/bin, but fedora and suse pip like to
|
||||||
|
# install things in /usr/bin
|
||||||
|
iniset $GYAN_CONF DEFAULT bindir "/usr/bin"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "$GYAN_STATE_PATH" ]; then
|
||||||
|
iniset $GYAN_CONF DEFAULT state_path "$GYAN_STATE_PATH"
|
||||||
|
iniset $GYAN_CONF oslo_concurrency lock_path "$GYAN_STATE_PATH"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$SYSLOG" != "False" ]; then
|
||||||
|
iniset $GYAN_CONF DEFAULT use_syslog "True"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Format logging
|
||||||
|
if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then
|
||||||
|
setup_colorized_logging $GYAN_CONF DEFAULT
|
||||||
|
else
|
||||||
|
# Show user_name and project_name instead of user_id and project_id
|
||||||
|
iniset $GYAN_CONF DEFAULT logging_context_format_string "%(asctime)s.%(msecs)03d %(levelname)s %(name)s [%(request_id)s %(user_name)s %(project_name)s] %(instance)s%(message)s"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Register SSL certificates if provided
|
||||||
|
if is_ssl_enabled_service gyan; then
|
||||||
|
ensure_certificates gyan
|
||||||
|
|
||||||
|
iniset $GYAN_CONF DEFAULT ssl_cert_file "$GYAN_SSL_CERT"
|
||||||
|
iniset $GYAN_CONF DEFAULT ssl_key_file "$GYAN_SSL_KEY"
|
||||||
|
|
||||||
|
iniset $GYAN_CONF DEFAULT enabled_ssl_apis "$GYAN_ENABLED_APIS"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function create_api_paste_conf {
|
||||||
|
# copy api_paste.ini
|
||||||
|
cp $GYAN_DIR/etc/gyan/api-paste.ini $GYAN_API_PASTE
|
||||||
|
}
|
||||||
|
|
||||||
|
# create_gyan_cache_dir() - Part of the init_GYAN() process
|
||||||
|
function create_gyan_cache_dir {
|
||||||
|
# Create cache dir
|
||||||
|
sudo mkdir -p $GYAN_AUTH_CACHE_DIR
|
||||||
|
sudo chown $STACK_USER $GYAN_AUTH_CACHE_DIR
|
||||||
|
rm -f $GYAN_AUTH_CACHE_DIR/*
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# init_gyan() - Initialize databases, etc.
|
||||||
|
function init_gyan {
|
||||||
|
# Only do this step once on the API node for an entire cluster.
|
||||||
|
if is_service_enabled gyan-api; then
|
||||||
|
if is_service_enabled $DATABASE_BACKENDS; then
|
||||||
|
# (Re)create gyan database
|
||||||
|
recreate_database gyan
|
||||||
|
|
||||||
|
# Migrate gyan database
|
||||||
|
$GYAN_BIN_DIR/gyan-db-manage upgrade
|
||||||
|
fi
|
||||||
|
|
||||||
|
if is_service_enabled gyan-etcd; then
|
||||||
|
install_etcd_server
|
||||||
|
fi
|
||||||
|
create_gyan_cache_dir
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# install_gyanclient() - Collect source and prepare
|
||||||
|
function install_gyanclient {
|
||||||
|
if use_library_from_git "python-gyanclient"; then
|
||||||
|
git_clone_by_name "python-gyanclient"
|
||||||
|
setup_dev_lib "python-gyanclient"
|
||||||
|
sudo install -D -m 0644 -o $STACK_USER {${GITDIR["python-gyanclient"]}/tools/,/etc/bash_completion.d/}gyan.bash_completion
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# install_gyan() - Collect source and prepare
|
||||||
|
function install_gyan {
|
||||||
|
git_clone $GYAN_REPO $GYAN_DIR $GYAN_BRANCH
|
||||||
|
setup_develop $GYAN_DIR
|
||||||
|
}
|
||||||
|
|
||||||
|
# start_gyan_api() - Start the API process ahead of other things
|
||||||
|
function start_gyan_api {
|
||||||
|
# Get right service port for testing
|
||||||
|
local service_port=$GYAN_SERVICE_PORT
|
||||||
|
local service_protocol=$GYAN_SERVICE_PROTOCOL
|
||||||
|
if is_service_enabled tls-proxy; then
|
||||||
|
service_port=$GYAN_SERVICE_PORT_INT
|
||||||
|
service_protocol="http"
|
||||||
|
fi
|
||||||
|
|
||||||
|
local gyan_url
|
||||||
|
if [ "$GYAN_USE_UWSGI" == "True" ]; then
|
||||||
|
run_process gyan-api "$GYAN_BIN_DIR/uwsgi --procname-prefix gyan-api --ini $GYAN_UWSGI_CONF"
|
||||||
|
gyan_url=$service_protocol://$GYAN_SERVICE_HOST/ml-infra
|
||||||
|
else
|
||||||
|
run_process gyan-api "$GYAN_BIN_DIR/gyan-api"
|
||||||
|
gyan_url=$service_protocol://$GYAN_SERVICE_HOST:$service_port
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Waiting for gyan-api to start..."
|
||||||
|
if ! wait_for_service $SERVICE_TIMEOUT $gyan_url; then
|
||||||
|
die $LINENO "gyan-api did not start"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Start proxies if enabled
|
||||||
|
if is_service_enabled tls-proxy; then
|
||||||
|
start_tls_proxy '*' $GYAN_SERVICE_PORT $GYAN_SERVICE_HOST $GYAN_SERVICE_PORT_INT &
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# start_gyan_compute() - Start Gyan compute agent
|
||||||
|
function start_gyan_compute {
|
||||||
|
echo "Start gyan compute..."
|
||||||
|
run_process gyan-compute "$GYAN_BIN_DIR/gyan-compute"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# start_gyan() - Start running processes, including screen
|
||||||
|
function start_gyan {
|
||||||
|
|
||||||
|
# ``run_process`` checks ``is_service_enabled``, it is not needed here
|
||||||
|
start_gyan_api
|
||||||
|
start_gyan_compute
|
||||||
|
}
|
||||||
|
|
||||||
|
# stop_gyan() - Stop running processes (non-screen)
|
||||||
|
function stop_gyan {
|
||||||
|
|
||||||
|
if [ "$GYAN_USE_UWSGI" == "True" ]; then
|
||||||
|
disable_apache_site gyan
|
||||||
|
restart_apache_server
|
||||||
|
else
|
||||||
|
stop_process gyan-api
|
||||||
|
fi
|
||||||
|
stop_process gyan-compute
|
||||||
|
}
|
||||||
|
|
||||||
|
# Restore xtrace
|
||||||
|
$XTRACE
|
||||||
13
devstack/local.conf.sample
Normal file
13
devstack/local.conf.sample
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
[[local|localrc]]
|
||||||
|
HOST_IP=10.0.0.11 # change this to your IP address
|
||||||
|
DATABASE_PASSWORD=password
|
||||||
|
RABBIT_PASSWORD=password
|
||||||
|
SERVICE_TOKEN=password
|
||||||
|
SERVICE_PASSWORD=password
|
||||||
|
ADMIN_PASSWORD=password
|
||||||
|
enable_plugin gyan https://git.openstack.org/openstack/gyan
|
||||||
|
|
||||||
|
# install python-gyanclient from git
|
||||||
|
LIBS_FROM_GIT="python-gyanclient"
|
||||||
|
|
||||||
|
|
||||||
18
devstack/local.conf.subnode.sample
Normal file
18
devstack/local.conf.subnode.sample
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
[[local|localrc]]
|
||||||
|
HOST_IP=10.0.0.31 # change this to your IP address
|
||||||
|
DATABASE_PASSWORD=password
|
||||||
|
RABBIT_PASSWORD=password
|
||||||
|
SERVICE_TOKEN=password
|
||||||
|
SERVICE_PASSWORD=password
|
||||||
|
ADMIN_PASSWORD=password
|
||||||
|
enable_plugin gyan https://git.openstack.org/openstack/gyan
|
||||||
|
|
||||||
|
|
||||||
|
# Following is for multi host settings
|
||||||
|
MULTI_HOST=True
|
||||||
|
SERVICE_HOST=10.0.0.11 # change this to controller's IP address
|
||||||
|
DATABASE_TYPE=mysql
|
||||||
|
MYSQL_HOST=$SERVICE_HOST
|
||||||
|
RABBIT_HOST=$SERVICE_HOST
|
||||||
|
|
||||||
|
ENABLED_SERVICES=gyan-compute
|
||||||
47
devstack/plugin.sh
Executable file
47
devstack/plugin.sh
Executable file
@ -0,0 +1,47 @@
|
|||||||
|
# gyan - Devstack extras script to install gyan
|
||||||
|
|
||||||
|
# Save trace setting
|
||||||
|
XTRACE=$(set +o | grep xtrace)
|
||||||
|
set -o xtrace
|
||||||
|
|
||||||
|
echo_summary "gyan's plugin.sh was called..."
|
||||||
|
source $DEST/gyan/devstack/lib/gyan
|
||||||
|
(set -o posix; set)
|
||||||
|
|
||||||
|
if is_service_enabled gyan-api gyan-compute; then
|
||||||
|
if [[ "$1" == "stack" && "$2" == "install" ]]; then
|
||||||
|
echo_summary "Installing gyan"
|
||||||
|
install_gyan
|
||||||
|
|
||||||
|
install_gyanclient
|
||||||
|
cleanup_gyan
|
||||||
|
|
||||||
|
elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
|
||||||
|
echo_summary "Configuring gyan"
|
||||||
|
configure_gyan
|
||||||
|
|
||||||
|
if is_service_enabled key; then
|
||||||
|
create_gyan_accounts
|
||||||
|
fi
|
||||||
|
|
||||||
|
elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
|
||||||
|
# Initialize gyan
|
||||||
|
init_gyan
|
||||||
|
|
||||||
|
# Start the gyan API and gyan compute
|
||||||
|
echo_summary "Starting gyan"
|
||||||
|
start_gyan
|
||||||
|
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$1" == "unstack" ]]; then
|
||||||
|
stop_gyan
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$1" == "clean" ]]; then
|
||||||
|
cleanup_gyan
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Restore xtrace
|
||||||
|
$XTRACE
|
||||||
24
devstack/settings
Normal file
24
devstack/settings
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
# Devstack settings
|
||||||
|
|
||||||
|
## Modify to your environment
|
||||||
|
# FLOATING_RANGE=192.168.1.224/27
|
||||||
|
# PUBLIC_NETWORK_GATEWAY=192.168.1.225
|
||||||
|
# PUBLIC_INTERFACE=em1
|
||||||
|
# FIXED_RANGE=10.0.0.0/24
|
||||||
|
## Log all output to files
|
||||||
|
# LOGFILE=$HOME/devstack.log
|
||||||
|
## Neutron settings
|
||||||
|
# Q_USE_SECGROUP=True
|
||||||
|
# ENABLE_TENANT_VLANS=True
|
||||||
|
# TENANT_VLAN_RANGE=
|
||||||
|
# PHYSICAL_NETWORK=public
|
||||||
|
# OVS_PHYSICAL_BRIDGE=br-ex
|
||||||
|
|
||||||
|
# Enable Gyan services
|
||||||
|
if [[ ${HOST_IP} == ${SERVICE_HOST} ]]; then
|
||||||
|
enable_service gyan-api
|
||||||
|
enable_service gyan-compute
|
||||||
|
else
|
||||||
|
enable_service gyan-compute
|
||||||
|
fi
|
||||||
|
|
||||||
41
etc/apache2/gyan.conf.template
Normal file
41
etc/apache2/gyan.conf.template
Normal file
@ -0,0 +1,41 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
# This is an example Apache2 configuration file for using the
|
||||||
|
# gyan API through mod_wsgi.
|
||||||
|
|
||||||
|
# Note: If you are using a Debian-based system then the paths
|
||||||
|
# "/var/log/httpd" and "/var/run/httpd" will use "apache2" instead
|
||||||
|
# of "httpd".
|
||||||
|
#
|
||||||
|
# The number of processes and threads is an example only and should
|
||||||
|
# be adjusted according to local requirements.
|
||||||
|
|
||||||
|
Listen %PUBLICPORT%
|
||||||
|
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D(us)" gyan_combined
|
||||||
|
|
||||||
|
<VirtualHost *:%PUBLICPORT%>
|
||||||
|
WSGIDaemonProcess gyan-api user=%USER% processes=5 threads=1 display-name=%{GROUP}
|
||||||
|
WSGIScriptAlias / %PUBLICWSGI%
|
||||||
|
WSGIProcessGroup gyan-api
|
||||||
|
ErrorLogFormat "%M"
|
||||||
|
ErrorLog /var/log/%APACHE_NAME%/gyan_api.log
|
||||||
|
LogLevel info
|
||||||
|
CustomLog /var/log/%APACHE_NAME%/gyan_access.log gyan_combined
|
||||||
|
|
||||||
|
<Directory /opt/stack/gyan/gyan/api>
|
||||||
|
WSGIProcessGroup gyan-api
|
||||||
|
WSGIApplicationGroup %{GLOBAL}
|
||||||
|
AllowOverride All
|
||||||
|
Require all granted
|
||||||
|
</Directory>
|
||||||
|
</VirtualHost>
|
||||||
19
etc/gyan/api-paste.ini
Normal file
19
etc/gyan/api-paste.ini
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
[pipeline:main]
|
||||||
|
pipeline = cors request_id osprofiler authtoken api_v1
|
||||||
|
|
||||||
|
[app:api_v1]
|
||||||
|
paste.app_factory = gyan.api.app:app_factory
|
||||||
|
|
||||||
|
[filter:authtoken]
|
||||||
|
acl_public_routes = /, /v1
|
||||||
|
paste.filter_factory = gyan.api.middleware.auth_token:AuthTokenMiddleware.factory
|
||||||
|
|
||||||
|
[filter:osprofiler]
|
||||||
|
paste.filter_factory = gyan.common.profiler:WsgiMiddleware.factory
|
||||||
|
|
||||||
|
[filter:request_id]
|
||||||
|
paste.filter_factory = oslo_middleware:RequestId.factory
|
||||||
|
|
||||||
|
[filter:cors]
|
||||||
|
paste.filter_factory = oslo_middleware.cors:filter_factory
|
||||||
|
oslo_config_project = gyan
|
||||||
14
etc/gyan/gyan-config-generator.conf
Normal file
14
etc/gyan/gyan-config-generator.conf
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
[DEFAULT]
|
||||||
|
output_file = etc/gyan/gyan.conf.sample
|
||||||
|
wrap_width = 79
|
||||||
|
|
||||||
|
namespace = gyan.conf
|
||||||
|
namespace = keystonemiddleware.auth_token
|
||||||
|
namespace = oslo.concurrency
|
||||||
|
namespace = oslo.db
|
||||||
|
namespace = oslo.log
|
||||||
|
namespace = oslo.messaging
|
||||||
|
namespace = oslo.middleware.cors
|
||||||
|
namespace = oslo.policy
|
||||||
|
namespace = oslo.service.periodic_task
|
||||||
|
namespace = oslo.service.service
|
||||||
3
etc/gyan/gyan-policy-generator.conf
Normal file
3
etc/gyan/gyan-policy-generator.conf
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
[DEFAULT]
|
||||||
|
output_file = etc/gyan/policy.yaml.sample
|
||||||
|
namespace = gyan
|
||||||
27
etc/gyan/rootwrap.conf
Normal file
27
etc/gyan/rootwrap.conf
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
# Configuration for gyan-rootwrap
|
||||||
|
# This file should be owned by (and only-writable by) the root user
|
||||||
|
|
||||||
|
[DEFAULT]
|
||||||
|
# List of directories to load filter definitions from (separated by ',').
|
||||||
|
# These directories MUST all be only writable by root !
|
||||||
|
filters_path=/etc/gyan/rootwrap.d
|
||||||
|
|
||||||
|
# List of directories to search executables in, in case filters do not
|
||||||
|
# explicitely specify a full path (separated by ',')
|
||||||
|
# If not specified, defaults to system PATH environment variable.
|
||||||
|
# These directories MUST all be only writable by root !
|
||||||
|
exec_dirs=/sbin,/usr/sbin,/bin,/usr/bin,/usr/local/bin,/usr/local/sbin
|
||||||
|
|
||||||
|
# Enable logging to syslog
|
||||||
|
# Default value is False
|
||||||
|
use_syslog=False
|
||||||
|
|
||||||
|
# Which syslog facility to use.
|
||||||
|
# Valid values include auth, authpriv, syslog, local0, local1...
|
||||||
|
# Default value is 'syslog'
|
||||||
|
syslog_log_facility=syslog
|
||||||
|
|
||||||
|
# Which messages to log.
|
||||||
|
# INFO means log all usage
|
||||||
|
# ERROR means only log unsuccessful attempts
|
||||||
|
syslog_log_level=ERROR
|
||||||
8
etc/gyan/rootwrap.d/gyan.filters
Normal file
8
etc/gyan/rootwrap.d/gyan.filters
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
# gyan command filters
|
||||||
|
# This file should be owned by (and only-writeable by) the root user
|
||||||
|
|
||||||
|
[Filters]
|
||||||
|
# privileged/__init__.py: priv_context.PrivContext(default)
|
||||||
|
# This line ties the superuser privs with the config files, context name,
|
||||||
|
# and (implicitly) the actual python code invoked.
|
||||||
|
privsep-rootwrap: RegExpFilter, privsep-helper, root, privsep-helper, --config-file, /etc/(?!\.\.).*, --privsep_context, os_brick.privileged.default, --privsep_sock_path, /tmp/.*
|
||||||
1
gyan/MANIFEST.in
Normal file
1
gyan/MANIFEST.in
Normal file
@ -0,0 +1 @@
|
|||||||
|
recursive-include public *
|
||||||
0
gyan/api/__init__.py
Normal file
0
gyan/api/__init__.py
Normal file
67
gyan/api/app.py
Normal file
67
gyan/api/app.py
Normal file
@ -0,0 +1,67 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
from oslo_log import log
|
||||||
|
from paste import deploy
|
||||||
|
import pecan
|
||||||
|
|
||||||
|
from gyan.api import config as api_config
|
||||||
|
from gyan.api import middleware
|
||||||
|
from gyan.common import config as common_config
|
||||||
|
import gyan.conf
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def get_pecan_config():
|
||||||
|
# Set up the pecan configuration
|
||||||
|
filename = api_config.__file__.replace('.pyc', '.py')
|
||||||
|
return pecan.configuration.conf_from_file(filename)
|
||||||
|
|
||||||
|
|
||||||
|
def setup_app(config=None):
|
||||||
|
if not config:
|
||||||
|
config = get_pecan_config()
|
||||||
|
|
||||||
|
app_conf = dict(config.app)
|
||||||
|
common_config.set_config_defaults()
|
||||||
|
|
||||||
|
app = pecan.make_app(
|
||||||
|
app_conf.pop('root'),
|
||||||
|
logging=getattr(config, 'logging', {}),
|
||||||
|
wrap_app=middleware.ParsableErrorMiddleware,
|
||||||
|
**app_conf
|
||||||
|
)
|
||||||
|
|
||||||
|
return app
|
||||||
|
|
||||||
|
|
||||||
|
def load_app():
|
||||||
|
cfg_file = None
|
||||||
|
cfg_path = CONF.api.api_paste_config
|
||||||
|
if not os.path.isabs(cfg_path):
|
||||||
|
cfg_file = CONF.find_file(cfg_path)
|
||||||
|
elif os.path.exists(cfg_path):
|
||||||
|
cfg_file = cfg_path
|
||||||
|
|
||||||
|
if not cfg_file:
|
||||||
|
raise cfg.ConfigFilesNotFoundError([CONF.api.api_paste_config])
|
||||||
|
LOG.info("Full WSGI config used: %s", cfg_file)
|
||||||
|
return deploy.loadapp("config:" + cfg_file)
|
||||||
|
|
||||||
|
|
||||||
|
def app_factory(global_config, **local_conf):
|
||||||
|
return setup_app()
|
||||||
19
gyan/api/app.wsgi
Normal file
19
gyan/api/app.wsgi
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""Use this file for deploying the API under mod_wsgi.
|
||||||
|
See https://pecan.readthedocs.org/en/latest/deployment.html for details.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from gyan.api import wsgi
|
||||||
|
|
||||||
|
application = wsgi.init_application()
|
||||||
25
gyan/api/config.py
Normal file
25
gyan/api/config.py
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from gyan.api import hooks
|
||||||
|
|
||||||
|
# Pecan Application Configurations
|
||||||
|
app = {
|
||||||
|
'root': 'gyan.api.controllers.root.RootController',
|
||||||
|
'modules': ['gyan'],
|
||||||
|
'hooks': [
|
||||||
|
hooks.ContextHook(),
|
||||||
|
hooks.NoExceptionTracebackHook(),
|
||||||
|
hooks.RPCHook(),
|
||||||
|
],
|
||||||
|
'debug': True,
|
||||||
|
}
|
||||||
0
gyan/api/controllers/__init__.py
Normal file
0
gyan/api/controllers/__init__.py
Normal file
233
gyan/api/controllers/base.py
Normal file
233
gyan/api/controllers/base.py
Normal file
@ -0,0 +1,233 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import operator
|
||||||
|
import six
|
||||||
|
|
||||||
|
import pecan
|
||||||
|
from pecan import rest
|
||||||
|
from webob import exc
|
||||||
|
from gyan.api.controllers import versions
|
||||||
|
from gyan.api import versioned_method
|
||||||
|
from gyan.common import exception
|
||||||
|
from gyan.common.i18n import _
|
||||||
|
|
||||||
|
|
||||||
|
# name of attribute to keep version method information
|
||||||
|
VER_METHOD_ATTR = 'versioned_methods'
|
||||||
|
|
||||||
|
|
||||||
|
class APIBase(object):
|
||||||
|
|
||||||
|
def __init__(self, **kwargs):
|
||||||
|
for field in self.fields:
|
||||||
|
if field in kwargs:
|
||||||
|
value = kwargs[field]
|
||||||
|
setattr(self, field, value)
|
||||||
|
|
||||||
|
def __setattr__(self, field, value):
|
||||||
|
super(APIBase, self).__setattr__(field, value)
|
||||||
|
|
||||||
|
def as_dict(self):
|
||||||
|
"""Render this object as a dict of its fields."""
|
||||||
|
return {f: getattr(self, f)
|
||||||
|
for f in self.fields
|
||||||
|
if hasattr(self, f)}
|
||||||
|
|
||||||
|
def __json__(self):
|
||||||
|
return self.as_dict()
|
||||||
|
|
||||||
|
def unset_fields_except(self, except_list=None):
|
||||||
|
"""Unset fields so they don't appear in the message body.
|
||||||
|
|
||||||
|
:param except_list: A list of fields that won't be touched.
|
||||||
|
|
||||||
|
"""
|
||||||
|
if except_list is None:
|
||||||
|
except_list = []
|
||||||
|
|
||||||
|
for k in self.as_dict():
|
||||||
|
if k not in except_list:
|
||||||
|
setattr(self, k, None)
|
||||||
|
|
||||||
|
|
||||||
|
class ControllerMetaclass(type):
|
||||||
|
"""Controller metaclass.
|
||||||
|
|
||||||
|
This metaclass automates the task of assembling a dictionary
|
||||||
|
mapping action keys to method names.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __new__(mcs, name, bases, cls_dict):
|
||||||
|
"""Adds version function dictionary to the class."""
|
||||||
|
|
||||||
|
versioned_methods = None
|
||||||
|
|
||||||
|
for base in bases:
|
||||||
|
if base.__name__ == "Controller":
|
||||||
|
# NOTE(cyeoh): This resets the VER_METHOD_ATTR attribute
|
||||||
|
# between API controller class creations. This allows us
|
||||||
|
# to use a class decorator on the API methods that doesn't
|
||||||
|
# require naming explicitly what method is being versioned as
|
||||||
|
# it can be implicit based on the method decorated. It is a bit
|
||||||
|
# ugly.
|
||||||
|
if VER_METHOD_ATTR in base.__dict__:
|
||||||
|
versioned_methods = getattr(base, VER_METHOD_ATTR)
|
||||||
|
delattr(base, VER_METHOD_ATTR)
|
||||||
|
|
||||||
|
if versioned_methods:
|
||||||
|
cls_dict[VER_METHOD_ATTR] = versioned_methods
|
||||||
|
|
||||||
|
return super(ControllerMetaclass, mcs).__new__(mcs, name, bases,
|
||||||
|
cls_dict)
|
||||||
|
|
||||||
|
|
||||||
|
@six.add_metaclass(ControllerMetaclass)
|
||||||
|
class Controller(rest.RestController):
|
||||||
|
"""Base Rest Controller"""
|
||||||
|
|
||||||
|
@pecan.expose('json')
|
||||||
|
def _no_version_match(self, *args, **kwargs):
|
||||||
|
from pecan import request
|
||||||
|
|
||||||
|
raise exc.HTTPNotAcceptable(_(
|
||||||
|
"Version %(ver)s was requested but the requested API is not "
|
||||||
|
"supported for this version.") % {'ver': request.version})
|
||||||
|
|
||||||
|
def __getattribute__(self, key):
|
||||||
|
|
||||||
|
def version_select():
|
||||||
|
"""Select the correct method based on version
|
||||||
|
|
||||||
|
@return: Returns the correct versioned method
|
||||||
|
@raises: HTTPNotAcceptable if there is no method which
|
||||||
|
matches the name and version constraints
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pecan import request
|
||||||
|
ver = request.version
|
||||||
|
|
||||||
|
func_list = self.versioned_methods[key]
|
||||||
|
for func in func_list:
|
||||||
|
if ver.matches(func.start_version, func.end_version):
|
||||||
|
return func.func
|
||||||
|
|
||||||
|
return self._no_version_match
|
||||||
|
|
||||||
|
try:
|
||||||
|
version_meth_dict = object.__getattribute__(self, VER_METHOD_ATTR)
|
||||||
|
except AttributeError:
|
||||||
|
# No versioning on this class
|
||||||
|
return object.__getattribute__(self, key)
|
||||||
|
if version_meth_dict and key in version_meth_dict:
|
||||||
|
return version_select().__get__(self, self.__class__)
|
||||||
|
|
||||||
|
return object.__getattribute__(self, key)
|
||||||
|
|
||||||
|
# NOTE: This decorator MUST appear first (the outermost
|
||||||
|
# decorator) on an API method for it to work correctly
|
||||||
|
@classmethod
|
||||||
|
def api_version(cls, min_ver, max_ver=None):
|
||||||
|
"""Decorator for versioning api methods.
|
||||||
|
|
||||||
|
Add the decorator to any pecan method that has been exposed.
|
||||||
|
This decorator will store the method, min version, and max
|
||||||
|
version in a list for each api. It will check that there is no
|
||||||
|
overlap between versions and methods. When the api is called the
|
||||||
|
controller will use the list for each api to determine which
|
||||||
|
method to call.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
@base.Controller.api_version("1.1", "1.2")
|
||||||
|
def get_one(self, ml_model_id):
|
||||||
|
{...code for versions 1.1 to 1.2...}
|
||||||
|
|
||||||
|
@base.Controller.api_version("1.3")
|
||||||
|
def get_one(self, ml_model_id):
|
||||||
|
{...code for versions 1.3 to latest}
|
||||||
|
|
||||||
|
@min_ver: string representing minimum version
|
||||||
|
@max_ver: optional string representing maximum version
|
||||||
|
@raises: ApiVersionsIntersect if an version overlap is found between
|
||||||
|
method versions.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def decorator(f):
|
||||||
|
obj_min_ver = versions.Version('', '', '', min_ver)
|
||||||
|
if max_ver:
|
||||||
|
obj_max_ver = versions.Version('', '', '', max_ver)
|
||||||
|
else:
|
||||||
|
obj_max_ver = versions.Version('', '', '',
|
||||||
|
versions.CURRENT_MAX_VER)
|
||||||
|
|
||||||
|
# Add to list of versioned methods registered
|
||||||
|
func_name = f.__name__
|
||||||
|
new_func = versioned_method.VersionedMethod(
|
||||||
|
func_name, obj_min_ver, obj_max_ver, f)
|
||||||
|
|
||||||
|
func_dict = getattr(cls, VER_METHOD_ATTR, {})
|
||||||
|
if not func_dict:
|
||||||
|
setattr(cls, VER_METHOD_ATTR, func_dict)
|
||||||
|
|
||||||
|
func_list = func_dict.get(func_name, [])
|
||||||
|
if not func_list:
|
||||||
|
func_dict[func_name] = func_list
|
||||||
|
func_list.append(new_func)
|
||||||
|
|
||||||
|
is_intersect = Controller.check_for_versions_intersection(
|
||||||
|
func_list)
|
||||||
|
|
||||||
|
if is_intersect:
|
||||||
|
raise exception.ApiVersionsIntersect(
|
||||||
|
name=new_func.name,
|
||||||
|
min_ver=new_func.start_version,
|
||||||
|
max_ver=new_func.end_version
|
||||||
|
)
|
||||||
|
|
||||||
|
# Ensure the list is sorted by minimum version (reversed)
|
||||||
|
# so later when we work through the list in order we find
|
||||||
|
# the method which has the latest version which supports
|
||||||
|
# the version requested.
|
||||||
|
func_list.sort(key=lambda f: f.start_version, reverse=True)
|
||||||
|
|
||||||
|
return f
|
||||||
|
|
||||||
|
return decorator
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def check_for_versions_intersection(func_list):
|
||||||
|
"""Determines whether function list intersections
|
||||||
|
|
||||||
|
General algorithm:
|
||||||
|
https://en.wikipedia.org/wiki/Intersection_algorithm
|
||||||
|
|
||||||
|
:param func_list: list of VersionedMethod objects
|
||||||
|
:return: boolean
|
||||||
|
"""
|
||||||
|
|
||||||
|
pairs = []
|
||||||
|
counter = 0
|
||||||
|
|
||||||
|
for f in func_list:
|
||||||
|
pairs.append((f.start_version, 1))
|
||||||
|
pairs.append((f.end_version, -1))
|
||||||
|
|
||||||
|
pairs.sort(key=operator.itemgetter(1), reverse=True)
|
||||||
|
pairs.sort(key=operator.itemgetter(0))
|
||||||
|
|
||||||
|
for p in pairs:
|
||||||
|
counter += p[1]
|
||||||
|
|
||||||
|
if counter > 1:
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
35
gyan/api/controllers/link.py
Normal file
35
gyan/api/controllers/link.py
Normal file
@ -0,0 +1,35 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import pecan
|
||||||
|
|
||||||
|
|
||||||
|
def build_url(resource, resource_args, bookmark=False, base_url=None):
|
||||||
|
if base_url is None:
|
||||||
|
base_url = pecan.request.host_url
|
||||||
|
|
||||||
|
template = '%(url)s/%(res)s' if bookmark else '%(url)s/v1/%(res)s'
|
||||||
|
# FIXME(tbh): I'm getting a 404 when doing a GET on
|
||||||
|
# a nested resource that the URL ends with a '/'.
|
||||||
|
# https://groups.google.com/forum/#!topic/pecan-dev/QfSeviLg5qs
|
||||||
|
template += '%(args)s' if resource_args.startswith('?') else '/%(args)s'
|
||||||
|
return template % {'url': base_url, 'res': resource, 'args': resource_args}
|
||||||
|
|
||||||
|
|
||||||
|
def make_link(rel_name, url, resource, resource_args,
|
||||||
|
bookmark=False, type=None):
|
||||||
|
href = build_url(resource, resource_args,
|
||||||
|
bookmark=bookmark, base_url=url)
|
||||||
|
if type is None:
|
||||||
|
return {'href': href, 'rel': rel_name}
|
||||||
|
else:
|
||||||
|
return {'href': href, 'rel': rel_name, 'type': type}
|
||||||
97
gyan/api/controllers/root.py
Normal file
97
gyan/api/controllers/root.py
Normal file
@ -0,0 +1,97 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import pecan
|
||||||
|
from pecan import rest
|
||||||
|
|
||||||
|
from gyan.api.controllers import base
|
||||||
|
from gyan.api.controllers import link
|
||||||
|
from gyan.api.controllers import v1
|
||||||
|
from gyan.api.controllers import versions
|
||||||
|
|
||||||
|
|
||||||
|
class Version(base.APIBase):
|
||||||
|
"""An API version representation."""
|
||||||
|
|
||||||
|
fields = (
|
||||||
|
'id',
|
||||||
|
'links',
|
||||||
|
'status',
|
||||||
|
'max_version',
|
||||||
|
'min_version'
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def convert(id, status, max, min):
|
||||||
|
version = Version()
|
||||||
|
version.id = id
|
||||||
|
version.links = [link.make_link('self', pecan.request.host_url,
|
||||||
|
id, '', bookmark=True)]
|
||||||
|
version.status = status
|
||||||
|
version.max_version = max
|
||||||
|
version.min_version = min
|
||||||
|
return version
|
||||||
|
|
||||||
|
|
||||||
|
class Root(base.APIBase):
|
||||||
|
|
||||||
|
fields = (
|
||||||
|
'name',
|
||||||
|
'description',
|
||||||
|
'versions',
|
||||||
|
'default_version',
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def convert():
|
||||||
|
root = Root()
|
||||||
|
root.name = "OpenStack Gyan API"
|
||||||
|
root.description = ("Gyan is an OpenStack project which aims to "
|
||||||
|
"provide ML infra service.")
|
||||||
|
|
||||||
|
root.versions = [Version.convert('v1', "CURRENT",
|
||||||
|
versions.CURRENT_MAX_VER,
|
||||||
|
versions.BASE_VER)]
|
||||||
|
root.default_version = Version.convert('v1', "CURRENT",
|
||||||
|
versions.CURRENT_MAX_VER,
|
||||||
|
versions.BASE_VER)
|
||||||
|
return root
|
||||||
|
|
||||||
|
|
||||||
|
class RootController(rest.RestController):
|
||||||
|
|
||||||
|
_versions = ['v1']
|
||||||
|
"""All supported API versions"""
|
||||||
|
|
||||||
|
_default_version = 'v1'
|
||||||
|
"""The default API version"""
|
||||||
|
|
||||||
|
v1 = v1.Controller()
|
||||||
|
|
||||||
|
@pecan.expose('json')
|
||||||
|
def get(self):
|
||||||
|
# NOTE: The reason why convert() it's being called for every
|
||||||
|
# request is because we need to get the host url from
|
||||||
|
# the request object to make the links.
|
||||||
|
return Root.convert()
|
||||||
|
|
||||||
|
@pecan.expose()
|
||||||
|
def _route(self, args):
|
||||||
|
"""Overrides the default routing behavior.
|
||||||
|
|
||||||
|
It redirects the request to the default version of the gyan API
|
||||||
|
if the version number is not specified in the url.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if args[0] and args[0] not in self._versions:
|
||||||
|
args = [self._default_version] + args
|
||||||
|
return super(RootController, self)._route(args)
|
||||||
155
gyan/api/controllers/v1/__init__.py
Normal file
155
gyan/api/controllers/v1/__init__.py
Normal file
@ -0,0 +1,155 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""
|
||||||
|
Version 1 of the Gyan API
|
||||||
|
|
||||||
|
NOTE: IN PROGRESS AND NOT FULLY IMPLEMENTED.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from oslo_log import log as logging
|
||||||
|
import pecan
|
||||||
|
|
||||||
|
from gyan.api.controllers import base as controllers_base
|
||||||
|
from gyan.api.controllers import link
|
||||||
|
from gyan.api.controllers.v1 import hosts as host_controller
|
||||||
|
from gyan.api.controllers.v1 import ml_models as ml_model_controller
|
||||||
|
from gyan.api.controllers import versions as ver
|
||||||
|
from gyan.api import http_error
|
||||||
|
from gyan.common.i18n import _
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
BASE_VERSION = 1
|
||||||
|
|
||||||
|
MIN_VER_STR = '%s %s' % (ver.Version.service_string, ver.BASE_VER)
|
||||||
|
|
||||||
|
MAX_VER_STR = '%s %s' % (ver.Version.service_string, ver.CURRENT_MAX_VER)
|
||||||
|
|
||||||
|
MIN_VER = ver.Version({ver.Version.string: MIN_VER_STR},
|
||||||
|
MIN_VER_STR, MAX_VER_STR)
|
||||||
|
MAX_VER = ver.Version({ver.Version.string: MAX_VER_STR},
|
||||||
|
MIN_VER_STR, MAX_VER_STR)
|
||||||
|
|
||||||
|
|
||||||
|
class MediaType(controllers_base.APIBase):
|
||||||
|
"""A media type representation."""
|
||||||
|
|
||||||
|
fields = (
|
||||||
|
'base',
|
||||||
|
'type',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class V1(controllers_base.APIBase):
|
||||||
|
"""The representation of the version 1 of the API."""
|
||||||
|
|
||||||
|
fields = (
|
||||||
|
'id',
|
||||||
|
'media_types',
|
||||||
|
'links',
|
||||||
|
'hosts',
|
||||||
|
'ml_models'
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def convert():
|
||||||
|
v1 = V1()
|
||||||
|
v1.id = "v1"
|
||||||
|
v1.links = [link.make_link('self', pecan.request.host_url,
|
||||||
|
'v1', '', bookmark=True),
|
||||||
|
link.make_link('describedby',
|
||||||
|
'https://docs.openstack.org',
|
||||||
|
'developer/gyan/dev',
|
||||||
|
'api-spec-v1.html',
|
||||||
|
bookmark=True, type='text/html')]
|
||||||
|
v1.media_types = [MediaType(base='application/json',
|
||||||
|
type='application/vnd.openstack.gyan.v1+json')]
|
||||||
|
v1.hosts = [link.make_link('self', pecan.request.host_url,
|
||||||
|
'hosts', ''),
|
||||||
|
link.make_link('bookmark',
|
||||||
|
pecan.request.host_url,
|
||||||
|
'hosts', '',
|
||||||
|
bookmark=True)]
|
||||||
|
v1.ml_models = [link.make_link('self', pecan.request.host_url,
|
||||||
|
'ml_models', ''),
|
||||||
|
link.make_link('bookmark',
|
||||||
|
pecan.request.host_url,
|
||||||
|
'ml_models', '',
|
||||||
|
bookmark=True)]
|
||||||
|
return v1
|
||||||
|
|
||||||
|
|
||||||
|
class Controller(controllers_base.Controller):
|
||||||
|
"""Version 1 API controller root."""
|
||||||
|
|
||||||
|
hosts = host_controller.HostController()
|
||||||
|
ml_models = ml_model_controller.MLModelController()
|
||||||
|
|
||||||
|
@pecan.expose('json')
|
||||||
|
def get(self):
|
||||||
|
return V1.convert()
|
||||||
|
|
||||||
|
def _check_version(self, version, headers=None):
|
||||||
|
if headers is None:
|
||||||
|
headers = {}
|
||||||
|
# ensure that major version in the URL matches the header
|
||||||
|
if version.major != BASE_VERSION:
|
||||||
|
raise http_error.HTTPNotAcceptableAPIVersion(_(
|
||||||
|
"Mutually exclusive versions requested. Version %(ver)s "
|
||||||
|
"requested but not supported by this service. "
|
||||||
|
"The supported version range is: "
|
||||||
|
"[%(min)s, %(max)s].") % {'ver': version,
|
||||||
|
'min': MIN_VER_STR,
|
||||||
|
'max': MAX_VER_STR},
|
||||||
|
headers=headers,
|
||||||
|
max_version=str(MAX_VER),
|
||||||
|
min_version=str(MIN_VER))
|
||||||
|
# ensure the minor version is within the supported range
|
||||||
|
if version < MIN_VER or version > MAX_VER:
|
||||||
|
raise http_error.HTTPNotAcceptableAPIVersion(_(
|
||||||
|
"Version %(ver)s was requested but the minor version is not "
|
||||||
|
"supported by this service. The supported version range is: "
|
||||||
|
"[%(min)s, %(max)s].") % {'ver': version, 'min': MIN_VER_STR,
|
||||||
|
'max': MAX_VER_STR},
|
||||||
|
headers=headers,
|
||||||
|
max_version=str(MAX_VER),
|
||||||
|
min_version=str(MIN_VER))
|
||||||
|
|
||||||
|
@pecan.expose()
|
||||||
|
def _route(self, args):
|
||||||
|
version = ver.Version(
|
||||||
|
pecan.request.headers, MIN_VER_STR, MAX_VER_STR)
|
||||||
|
|
||||||
|
# Always set the basic version headers
|
||||||
|
pecan.response.headers[ver.Version.min_string] = MIN_VER_STR
|
||||||
|
pecan.response.headers[ver.Version.max_string] = MAX_VER_STR
|
||||||
|
pecan.response.headers[ver.Version.string] = " ".join(
|
||||||
|
[ver.Version.service_string, str(version)])
|
||||||
|
pecan.response.headers["vary"] = ver.Version.string
|
||||||
|
|
||||||
|
# assert that requested version is supported
|
||||||
|
self._check_version(version, pecan.response.headers)
|
||||||
|
pecan.request.version = version
|
||||||
|
if pecan.request.body:
|
||||||
|
msg = ("Processing request: url: %(url)s, %(method)s, "
|
||||||
|
"body: %(body)s" %
|
||||||
|
{'url': pecan.request.url,
|
||||||
|
'method': pecan.request.method,
|
||||||
|
'body': pecan.request.body})
|
||||||
|
LOG.debug(msg)
|
||||||
|
|
||||||
|
return super(Controller, self)._route(args)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ('Controller',)
|
||||||
41
gyan/api/controllers/v1/collection.py
Normal file
41
gyan/api/controllers/v1/collection.py
Normal file
@ -0,0 +1,41 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import pecan
|
||||||
|
|
||||||
|
from gyan.api.controllers import base
|
||||||
|
from gyan.api.controllers import link
|
||||||
|
|
||||||
|
|
||||||
|
class Collection(base.APIBase):
|
||||||
|
|
||||||
|
@property
|
||||||
|
def collection(self):
|
||||||
|
return getattr(self, self._type)
|
||||||
|
|
||||||
|
def has_next(self, limit):
|
||||||
|
"""Return whether collection has more items."""
|
||||||
|
return len(self.collection) and len(self.collection) == limit
|
||||||
|
|
||||||
|
def get_next(self, limit, url=None, **kwargs):
|
||||||
|
"""Return a link to the next subset of the collection."""
|
||||||
|
if not self.has_next(limit):
|
||||||
|
return None
|
||||||
|
|
||||||
|
resource_url = url or self._type
|
||||||
|
q_args = ''.join(['%s=%s&' % (key, kwargs[key]) for key in kwargs])
|
||||||
|
next_args = '?%(args)slimit=%(limit)d&marker=%(marker)s' % {
|
||||||
|
'args': q_args, 'limit': limit,
|
||||||
|
'marker': self.collection[-1]['uuid']}
|
||||||
|
|
||||||
|
return link.make_link('next', pecan.request.host_url,
|
||||||
|
resource_url, next_args)['href']
|
||||||
109
gyan/api/controllers/v1/hosts.py
Normal file
109
gyan/api/controllers/v1/hosts.py
Normal file
@ -0,0 +1,109 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import pecan
|
||||||
|
|
||||||
|
from gyan.api.controllers import base
|
||||||
|
from gyan.api.controllers.v1 import collection
|
||||||
|
from gyan.api.controllers.v1.views import hosts_view as view
|
||||||
|
from gyan.api import utils as api_utils
|
||||||
|
from gyan.common import exception
|
||||||
|
from gyan.common import policy
|
||||||
|
from gyan import objects
|
||||||
|
|
||||||
|
|
||||||
|
def _get_host(host_ident):
|
||||||
|
host = api_utils.get_resource('ComputeHost', host_ident)
|
||||||
|
if not host:
|
||||||
|
pecan.abort(404, ('Not found; the host you requested '
|
||||||
|
'does not exist.'))
|
||||||
|
|
||||||
|
return host
|
||||||
|
|
||||||
|
|
||||||
|
def check_policy_on_host(host, action):
|
||||||
|
context = pecan.request.context
|
||||||
|
policy.enforce(context, action, host, action=action)
|
||||||
|
|
||||||
|
|
||||||
|
class HostCollection(collection.Collection):
|
||||||
|
"""API representation of a collection of hosts."""
|
||||||
|
|
||||||
|
fields = {
|
||||||
|
'hosts',
|
||||||
|
'next'
|
||||||
|
}
|
||||||
|
|
||||||
|
"""A list containing compute host objects"""
|
||||||
|
|
||||||
|
def __init__(self, **kwargs):
|
||||||
|
super(HostCollection, self).__init__(**kwargs)
|
||||||
|
self._type = 'hosts'
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def convert_with_links(hosts, limit, url=None,
|
||||||
|
expand=False, **kwargs):
|
||||||
|
collection = HostCollection()
|
||||||
|
collection.hosts = [view.format_host(url, p) for p in hosts]
|
||||||
|
collection.next = collection.get_next(limit, url=url, **kwargs)
|
||||||
|
return collection
|
||||||
|
|
||||||
|
|
||||||
|
class HostController(base.Controller):
|
||||||
|
"""Host info controller"""
|
||||||
|
|
||||||
|
@pecan.expose('json')
|
||||||
|
@base.Controller.api_version("1.0")
|
||||||
|
@exception.wrap_pecan_controller_exception
|
||||||
|
def get_all(self, **kwargs):
|
||||||
|
"""Retrieve a list of hosts"""
|
||||||
|
context = pecan.request.context
|
||||||
|
policy.enforce(context, "host:get_all",
|
||||||
|
action="host:get_all")
|
||||||
|
return self._get_host_collection(**kwargs)
|
||||||
|
|
||||||
|
def _get_host_collection(self, **kwargs):
|
||||||
|
context = pecan.request.context
|
||||||
|
limit = api_utils.validate_limit(kwargs.get('limit'))
|
||||||
|
sort_dir = api_utils.validate_sort_dir(kwargs.get('sort_dir', 'asc'))
|
||||||
|
sort_key = kwargs.get('sort_key', 'hostname')
|
||||||
|
expand = kwargs.get('expand')
|
||||||
|
filters = None
|
||||||
|
marker_obj = None
|
||||||
|
resource_url = kwargs.get('resource_url')
|
||||||
|
marker = kwargs.get('marker')
|
||||||
|
if marker:
|
||||||
|
marker_obj = objects.ComputeHost.get_by_uuid(context, marker)
|
||||||
|
hosts = objects.ComputeHost.list(context,
|
||||||
|
limit,
|
||||||
|
marker_obj,
|
||||||
|
sort_key,
|
||||||
|
sort_dir,
|
||||||
|
filters=filters)
|
||||||
|
return HostCollection.convert_with_links(hosts, limit,
|
||||||
|
url=resource_url,
|
||||||
|
expand=expand,
|
||||||
|
sort_key=sort_key,
|
||||||
|
sort_dir=sort_dir)
|
||||||
|
|
||||||
|
@pecan.expose('json')
|
||||||
|
@base.Controller.api_version("1.0")
|
||||||
|
@exception.wrap_pecan_controller_exception
|
||||||
|
def get_one(self, host_ident):
|
||||||
|
"""Retrieve information about the given host.
|
||||||
|
|
||||||
|
:param host_ident: UUID or name of a host.
|
||||||
|
"""
|
||||||
|
context = pecan.request.context
|
||||||
|
policy.enforce(context, "host:get", action="host:get")
|
||||||
|
host = _get_host(host_ident)
|
||||||
|
return view.format_host(pecan.request.host_url, host)
|
||||||
289
gyan/api/controllers/v1/ml_models.py
Normal file
289
gyan/api/controllers/v1/ml_models.py
Normal file
@ -0,0 +1,289 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import shlex
|
||||||
|
|
||||||
|
from oslo_log import log as logging
|
||||||
|
from oslo_utils import strutils
|
||||||
|
from oslo_utils import uuidutils
|
||||||
|
import pecan
|
||||||
|
import six
|
||||||
|
|
||||||
|
from gyan.api.controllers import base
|
||||||
|
from gyan.api.controllers import link
|
||||||
|
from gyan.api.controllers.v1 import collection
|
||||||
|
from gyan.api.controllers.v1.schemas import ml_models as schema
|
||||||
|
from gyan.api.controllers.v1.views import ml_models_view as view
|
||||||
|
from gyan.api import utils as api_utils
|
||||||
|
from gyan.api import validation
|
||||||
|
from gyan.common import consts
|
||||||
|
from gyan.common import context as gyan_context
|
||||||
|
from gyan.common import exception
|
||||||
|
from gyan.common.i18n import _
|
||||||
|
from gyan.common.policies import ml_model as policies
|
||||||
|
from gyan.common import policy
|
||||||
|
from gyan.common import utils
|
||||||
|
import gyan.conf
|
||||||
|
from gyan import objects
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def check_policy_on_ml_model(ml_model, action):
|
||||||
|
context = pecan.request.context
|
||||||
|
policy.enforce(context, action, ml_model, action=action)
|
||||||
|
|
||||||
|
|
||||||
|
class MLModelCollection(collection.Collection):
|
||||||
|
"""API representation of a collection of ml models."""
|
||||||
|
|
||||||
|
fields = {
|
||||||
|
'ml_models',
|
||||||
|
'next'
|
||||||
|
}
|
||||||
|
|
||||||
|
"""A list containing ml models objects"""
|
||||||
|
|
||||||
|
def __init__(self, **kwargs):
|
||||||
|
super(MLModelCollection, self).__init__(**kwargs)
|
||||||
|
self._type = 'ml_models'
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def convert_with_links(rpc_ml_models, limit, url=None,
|
||||||
|
expand=False, **kwargs):
|
||||||
|
context = pecan.request.context
|
||||||
|
collection = MLModelCollection()
|
||||||
|
collection.ml_models = \
|
||||||
|
[view.format_ml_model(context, url, p.as_dict())
|
||||||
|
for p in rpc_ml_models]
|
||||||
|
collection.next = collection.get_next(limit, url=url, **kwargs)
|
||||||
|
return collection
|
||||||
|
|
||||||
|
|
||||||
|
class MLModelController(base.Controller):
|
||||||
|
"""Controller for MLModels."""
|
||||||
|
|
||||||
|
_custom_actions = {
|
||||||
|
'train': ['POST'],
|
||||||
|
'deploy': ['GET'],
|
||||||
|
'undeploy': ['GET']
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pecan.expose('json')
|
||||||
|
@exception.wrap_pecan_controller_exception
|
||||||
|
def get_all(self, **kwargs):
|
||||||
|
"""Retrieve a list of ml models.
|
||||||
|
|
||||||
|
"""
|
||||||
|
context = pecan.request.context
|
||||||
|
policy.enforce(context, "ml_model:get_all",
|
||||||
|
action="ml_model:get_all")
|
||||||
|
return self._get_ml_models_collection(**kwargs)
|
||||||
|
|
||||||
|
def _get_ml_models_collection(self, **kwargs):
|
||||||
|
context = pecan.request.context
|
||||||
|
if utils.is_all_projects(kwargs):
|
||||||
|
policy.enforce(context, "ml_model:get_all_all_projects",
|
||||||
|
action="ml_model:get_all_all_projects")
|
||||||
|
context.all_projects = True
|
||||||
|
kwargs.pop('all_projects', None)
|
||||||
|
limit = api_utils.validate_limit(kwargs.pop('limit', None))
|
||||||
|
sort_dir = api_utils.validate_sort_dir(kwargs.pop('sort_dir', 'asc'))
|
||||||
|
sort_key = kwargs.pop('sort_key', 'id')
|
||||||
|
resource_url = kwargs.pop('resource_url', None)
|
||||||
|
expand = kwargs.pop('expand', None)
|
||||||
|
|
||||||
|
ml_model_allowed_filters = ['name', 'status', 'project_id', 'user_id',
|
||||||
|
'type']
|
||||||
|
filters = {}
|
||||||
|
for filter_key in ml_model_allowed_filters:
|
||||||
|
if filter_key in kwargs:
|
||||||
|
policy_action = policies.MLMODEL % ('get_one:' + filter_key)
|
||||||
|
context.can(policy_action, might_not_exist=True)
|
||||||
|
filter_value = kwargs.pop(filter_key)
|
||||||
|
filters[filter_key] = filter_value
|
||||||
|
marker_obj = None
|
||||||
|
marker = kwargs.pop('marker', None)
|
||||||
|
if marker:
|
||||||
|
marker_obj = objects.ML_Model.get_by_uuid(context,
|
||||||
|
marker)
|
||||||
|
if kwargs:
|
||||||
|
unknown_params = [str(k) for k in kwargs]
|
||||||
|
msg = _("Unknown parameters: %s") % ", ".join(unknown_params)
|
||||||
|
raise exception.InvalidValue(msg)
|
||||||
|
|
||||||
|
ml_models = objects.ML_Model.list(context,
|
||||||
|
limit,
|
||||||
|
marker_obj,
|
||||||
|
sort_key,
|
||||||
|
sort_dir,
|
||||||
|
filters=filters)
|
||||||
|
return MLModelCollection.convert_with_links(ml_models, limit,
|
||||||
|
url=resource_url,
|
||||||
|
expand=expand,
|
||||||
|
sort_key=sort_key,
|
||||||
|
sort_dir=sort_dir)
|
||||||
|
|
||||||
|
@pecan.expose('json')
|
||||||
|
@exception.wrap_pecan_controller_exception
|
||||||
|
def get_one(self, ml_model_ident, **kwargs):
|
||||||
|
"""Retrieve information about the given ml_model.
|
||||||
|
|
||||||
|
:param ml_model_ident: UUID or name of a ml_model.
|
||||||
|
"""
|
||||||
|
context = pecan.request.context
|
||||||
|
if utils.is_all_projects(kwargs):
|
||||||
|
policy.enforce(context, "ml_model:get_one_all_projects",
|
||||||
|
action="ml_model:get_one_all_projects")
|
||||||
|
context.all_projects = True
|
||||||
|
ml_model = utils.get_ml_model(ml_model_ident)
|
||||||
|
check_policy_on_ml_model(ml_model.as_dict(), "ml_model:get_one")
|
||||||
|
if ml_model.node:
|
||||||
|
compute_api = pecan.request.compute_api
|
||||||
|
try:
|
||||||
|
ml_model = compute_api.ml_model_show(context, ml_model)
|
||||||
|
except exception.MLModelHostNotUp:
|
||||||
|
raise exception.ServerNotUsable
|
||||||
|
|
||||||
|
return view.format_ml_model(context, pecan.request.host_url,
|
||||||
|
ml_model.as_dict())
|
||||||
|
|
||||||
|
@base.Controller.api_version("1.0")
|
||||||
|
@pecan.expose('json')
|
||||||
|
@api_utils.enforce_content_types(['application/json'])
|
||||||
|
@exception.wrap_pecan_controller_exception
|
||||||
|
@validation.validate_query_param(pecan.request, schema.query_param_create)
|
||||||
|
@validation.validated(schema.ml_model_create)
|
||||||
|
def post(self, **ml_model_dict):
|
||||||
|
return self._do_post(**ml_model_dict)
|
||||||
|
|
||||||
|
|
||||||
|
def _do_post(self, **ml_model_dict):
|
||||||
|
"""Create or run a new ml model.
|
||||||
|
|
||||||
|
:param ml_model_dict: a ml_model within the request body.
|
||||||
|
"""
|
||||||
|
context = pecan.request.context
|
||||||
|
compute_api = pecan.request.compute_api
|
||||||
|
policy.enforce(context, "ml_model:create",
|
||||||
|
action="ml_model:create")
|
||||||
|
|
||||||
|
ml_model_dict['project_id'] = context.project_id
|
||||||
|
ml_model_dict['user_id'] = context.user_id
|
||||||
|
name = ml_model_dict.get('name')
|
||||||
|
ml_model_dict['name'] = name
|
||||||
|
|
||||||
|
ml_model_dict['status'] = consts.CREATING
|
||||||
|
extra_spec = {}
|
||||||
|
extra_spec['hints'] = ml_model_dict.get('hints', None)
|
||||||
|
new_ml_model = objects.ML_Model(context, **ml_model_dict)
|
||||||
|
new_ml_model.create(context)
|
||||||
|
|
||||||
|
compute_api.ml_model_create(context, new_ml_model, **kwargs)
|
||||||
|
# Set the HTTP Location Header
|
||||||
|
pecan.response.location = link.build_url('ml_models',
|
||||||
|
new_ml_model.uuid)
|
||||||
|
pecan.response.status = 202
|
||||||
|
return view.format_ml_model(context, pecan.request.node_url,
|
||||||
|
new_ml_model.as_dict())
|
||||||
|
|
||||||
|
|
||||||
|
@pecan.expose('json')
|
||||||
|
@exception.wrap_pecan_controller_exception
|
||||||
|
@validation.validated(schema.ml_model_update)
|
||||||
|
def patch(self, ml_model_ident, **patch):
|
||||||
|
"""Update an existing ml model.
|
||||||
|
|
||||||
|
:param ml_model_ident: UUID or name of a ml model.
|
||||||
|
:param patch: a json PATCH document to apply to this ml model.
|
||||||
|
"""
|
||||||
|
ml_model = utils.get_ml_model(ml_model_ident)
|
||||||
|
check_policy_on_ml_model(ml_model.as_dict(), "ml_model:update")
|
||||||
|
utils.validate_ml_model_state(ml_model, 'update')
|
||||||
|
context = pecan.request.context
|
||||||
|
compute_api = pecan.request.compute_api
|
||||||
|
ml_model = compute_api.ml_model_update(context, ml_model, patch)
|
||||||
|
return view.format_ml_model(context, pecan.request.node_url,
|
||||||
|
ml_model.as_dict())
|
||||||
|
|
||||||
|
|
||||||
|
@pecan.expose('json')
|
||||||
|
@exception.wrap_pecan_controller_exception
|
||||||
|
@validation.validate_query_param(pecan.request, schema.query_param_delete)
|
||||||
|
def delete(self, ml_model_ident, force=False, **kwargs):
|
||||||
|
"""Delete a ML Model.
|
||||||
|
|
||||||
|
:param ml_model_ident: UUID or Name of a ML Model.
|
||||||
|
:param force: If True, allow to force delete the ML Model.
|
||||||
|
"""
|
||||||
|
context = pecan.request.context
|
||||||
|
ml_model = utils.get_ml_model(ml_model_ident)
|
||||||
|
check_policy_on_ml_model(ml_model.as_dict(), "ml_model:delete")
|
||||||
|
try:
|
||||||
|
force = strutils.bool_from_string(force, strict=True)
|
||||||
|
except ValueError:
|
||||||
|
bools = ', '.join(strutils.TRUE_STRINGS + strutils.FALSE_STRINGS)
|
||||||
|
raise exception.InvalidValue(_('Valid force values are: %s')
|
||||||
|
% bools)
|
||||||
|
stop = kwargs.pop('stop', False)
|
||||||
|
try:
|
||||||
|
stop = strutils.bool_from_string(stop, strict=True)
|
||||||
|
except ValueError:
|
||||||
|
bools = ', '.join(strutils.TRUE_STRINGS + strutils.FALSE_STRINGS)
|
||||||
|
raise exception.InvalidValue(_('Valid stop values are: %s')
|
||||||
|
% bools)
|
||||||
|
compute_api = pecan.request.compute_api
|
||||||
|
if not force:
|
||||||
|
utils.validate_ml_model_state(ml_model, 'delete')
|
||||||
|
ml_model.status = consts.DELETING
|
||||||
|
if ml_model.node:
|
||||||
|
compute_api.ml_model_delete(context, ml_model, force)
|
||||||
|
else:
|
||||||
|
ml_model.destroy(context)
|
||||||
|
pecan.response.status = 204
|
||||||
|
|
||||||
|
|
||||||
|
@pecan.expose('json')
|
||||||
|
@exception.wrap_pecan_controller_exception
|
||||||
|
def deploy(self, ml_model_ident, **kwargs):
|
||||||
|
"""Deploy ML Model.
|
||||||
|
|
||||||
|
:param ml_model_ident: UUID or Name of a ML Model.
|
||||||
|
"""
|
||||||
|
ml_model = utils.get_ml_model(ml_model_ident)
|
||||||
|
check_policy_on_ml_model(ml_model.as_dict(), "ml_model:deploy")
|
||||||
|
utils.validate_ml_model_state(ml_model, 'deploy')
|
||||||
|
LOG.debug('Calling compute.ml_model_deploy with %s',
|
||||||
|
ml_model.uuid)
|
||||||
|
context = pecan.request.context
|
||||||
|
compute_api = pecan.request.compute_api
|
||||||
|
compute_api.ml_model_deploy(context, ml_model)
|
||||||
|
pecan.response.status = 202
|
||||||
|
|
||||||
|
@pecan.expose('json')
|
||||||
|
@exception.wrap_pecan_controller_exception
|
||||||
|
def undeploy(self, ml_model_ident, **kwargs):
|
||||||
|
"""Undeploy ML Model.
|
||||||
|
|
||||||
|
:param ml_model_ident: UUID or Name of a ML Model.
|
||||||
|
"""
|
||||||
|
ml_model = utils.get_ml_model(ml_model_ident)
|
||||||
|
check_policy_on_ml_model(ml_model.as_dict(), "ml_model:deploy")
|
||||||
|
utils.validate_ml_model_state(ml_model, 'undeploy')
|
||||||
|
LOG.debug('Calling compute.ml_model_deploy with %s',
|
||||||
|
ml_model.uuid)
|
||||||
|
context = pecan.request.context
|
||||||
|
compute_api = pecan.request.compute_api
|
||||||
|
compute_api.ml_model_undeploy(context, ml_model)
|
||||||
|
pecan.response.status = 202
|
||||||
0
gyan/api/controllers/v1/schemas/__init__.py
Normal file
0
gyan/api/controllers/v1/schemas/__init__.py
Normal file
49
gyan/api/controllers/v1/schemas/ml_models.py
Normal file
49
gyan/api/controllers/v1/schemas/ml_models.py
Normal file
@ -0,0 +1,49 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import copy
|
||||||
|
|
||||||
|
from gyan.api.controllers.v1.schemas import parameter_types
|
||||||
|
|
||||||
|
_ml_model_properties = {}
|
||||||
|
|
||||||
|
ml_model_create = {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': _ml_model_properties,
|
||||||
|
'required': ['name'],
|
||||||
|
'additionalProperties': False
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
query_param_create = {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'run': parameter_types.boolean_extended
|
||||||
|
},
|
||||||
|
'additionalProperties': False
|
||||||
|
}
|
||||||
|
|
||||||
|
ml_model_update = {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {},
|
||||||
|
'additionalProperties': False
|
||||||
|
}
|
||||||
|
|
||||||
|
query_param_delete = {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'force': parameter_types.boolean_extended,
|
||||||
|
'all_projects': parameter_types.boolean_extended,
|
||||||
|
'stop': parameter_types.boolean_extended
|
||||||
|
},
|
||||||
|
'additionalProperties': False
|
||||||
|
}
|
||||||
97
gyan/api/controllers/v1/schemas/parameter_types.py
Normal file
97
gyan/api/controllers/v1/schemas/parameter_types.py
Normal file
@ -0,0 +1,97 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import copy
|
||||||
|
import signal
|
||||||
|
import sys
|
||||||
|
import gyan.conf
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
|
||||||
|
non_negative_integer = {
|
||||||
|
'type': ['integer', 'string'],
|
||||||
|
'pattern': '^[0-9]*$', 'minimum': 0
|
||||||
|
}
|
||||||
|
|
||||||
|
positive_integer = {
|
||||||
|
'type': ['integer', 'string'],
|
||||||
|
'pattern': '^[0-9]*$', 'minimum': 1
|
||||||
|
}
|
||||||
|
|
||||||
|
boolean_extended = {
|
||||||
|
'type': ['boolean', 'string'],
|
||||||
|
'enum': [True, 'True', 'TRUE', 'true', '1', 'ON', 'On', 'on',
|
||||||
|
'YES', 'Yes', 'yes',
|
||||||
|
False, 'False', 'FALSE', 'false', '0', 'OFF', 'Off', 'off',
|
||||||
|
'NO', 'No', 'no'],
|
||||||
|
}
|
||||||
|
|
||||||
|
boolean = {
|
||||||
|
'type': ['boolean', 'string'],
|
||||||
|
'enum': [True, 'True', 'true', False, 'False', 'false'],
|
||||||
|
}
|
||||||
|
|
||||||
|
ml_model_name = {
|
||||||
|
'type': ['string', 'null'],
|
||||||
|
'minLength': 2,
|
||||||
|
'maxLength': 255,
|
||||||
|
'pattern': '^[a-zA-Z0-9][a-zA-Z0-9_.-]+$'
|
||||||
|
}
|
||||||
|
|
||||||
|
hex_uuid = {
|
||||||
|
'type': 'string',
|
||||||
|
'maxLength': 32,
|
||||||
|
'minLength': 32,
|
||||||
|
'pattern': '^[a-fA-F0-9]*$'
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
labels = {
|
||||||
|
'type': ['object', 'null']
|
||||||
|
}
|
||||||
|
|
||||||
|
hints = {
|
||||||
|
'type': ['object', 'null']
|
||||||
|
}
|
||||||
|
hostname = {
|
||||||
|
'type': ['string', 'null'],
|
||||||
|
'minLength': 2,
|
||||||
|
'maxLength': 63
|
||||||
|
}
|
||||||
|
|
||||||
|
repo = {
|
||||||
|
'type': 'string',
|
||||||
|
'minLength': 2,
|
||||||
|
'maxLength': 255,
|
||||||
|
'pattern': '[a-zA-Z0-9][a-zA-Z0-9_.-]'
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
string_ps_args = {
|
||||||
|
'type': ['string'],
|
||||||
|
'pattern': '[a-zA-Z- ,+]*'
|
||||||
|
}
|
||||||
|
|
||||||
|
str_and_int = {
|
||||||
|
'type': ['string', 'integer', 'null'],
|
||||||
|
}
|
||||||
|
|
||||||
|
hostname = {
|
||||||
|
'type': 'string', 'minLength': 1, 'maxLength': 255,
|
||||||
|
# NOTE: 'host' is defined in "services" table, and that
|
||||||
|
# means a hostname. The hostname grammar in RFC952 does
|
||||||
|
# not allow for underscores in hostnames. However, this
|
||||||
|
# schema allows them, because it sometimes occurs in
|
||||||
|
# real systems.
|
||||||
|
'pattern': '^[a-zA-Z0-9-._]*$',
|
||||||
|
}
|
||||||
50
gyan/api/controllers/v1/schemas/services.py
Normal file
50
gyan/api/controllers/v1/schemas/services.py
Normal file
@ -0,0 +1,50 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from gyan.api.controllers.v1.schemas import parameter_types
|
||||||
|
|
||||||
|
query_param_enable = {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'host': parameter_types.hostname,
|
||||||
|
'binary': {
|
||||||
|
'type': 'string', 'minLength': 1, 'maxLength': 255,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'additionalProperties': False
|
||||||
|
}
|
||||||
|
|
||||||
|
query_param_disable = {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'host': parameter_types.hostname,
|
||||||
|
'binary': {
|
||||||
|
'type': 'string', 'minLength': 1, 'maxLength': 255,
|
||||||
|
},
|
||||||
|
'disabled_reason': {
|
||||||
|
'type': 'string', 'minLength': 1, 'maxLength': 255,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'additionalProperties': False
|
||||||
|
}
|
||||||
|
|
||||||
|
query_param_force_down = {
|
||||||
|
'type': 'object',
|
||||||
|
'properties': {
|
||||||
|
'host': parameter_types.hostname,
|
||||||
|
'binary': {
|
||||||
|
'type': 'string', 'minLength': 1, 'maxLength': 255,
|
||||||
|
},
|
||||||
|
'forced_down': parameter_types.boolean
|
||||||
|
},
|
||||||
|
'additionalProperties': False
|
||||||
|
}
|
||||||
0
gyan/api/controllers/v1/views/__init__.py
Normal file
0
gyan/api/controllers/v1/views/__init__.py
Normal file
43
gyan/api/controllers/v1/views/hosts_view.py
Normal file
43
gyan/api/controllers/v1/views/hosts_view.py
Normal file
@ -0,0 +1,43 @@
|
|||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import itertools
|
||||||
|
|
||||||
|
from gyan.api.controllers import link
|
||||||
|
|
||||||
|
|
||||||
|
_basic_keys = (
|
||||||
|
'id',
|
||||||
|
'hostname',
|
||||||
|
'type',
|
||||||
|
'status'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def format_host(url, host):
|
||||||
|
def transform(key, value):
|
||||||
|
if key not in _basic_keys:
|
||||||
|
return
|
||||||
|
if key == 'id':
|
||||||
|
yield ('id', value)
|
||||||
|
yield ('links', [link.make_link(
|
||||||
|
'self', url, 'hosts', value),
|
||||||
|
link.make_link(
|
||||||
|
'bookmark', url,
|
||||||
|
'hosts', value,
|
||||||
|
bookmark=True)])
|
||||||
|
else:
|
||||||
|
yield (key, value)
|
||||||
|
|
||||||
|
return dict(itertools.chain.from_iterable(
|
||||||
|
transform(k, v) for k, v in host.as_dict().items()))
|
||||||
55
gyan/api/controllers/v1/views/ml_models_view.py
Normal file
55
gyan/api/controllers/v1/views/ml_models_view.py
Normal file
@ -0,0 +1,55 @@
|
|||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import itertools
|
||||||
|
|
||||||
|
from gyan.api.controllers import link
|
||||||
|
from gyan.common.policies import ml_model as policies
|
||||||
|
|
||||||
|
_basic_keys = (
|
||||||
|
'uuid',
|
||||||
|
'user_id',
|
||||||
|
'project_id',
|
||||||
|
'name',
|
||||||
|
'url',
|
||||||
|
'status',
|
||||||
|
'status_reason',
|
||||||
|
'task_state',
|
||||||
|
'labels',
|
||||||
|
'host',
|
||||||
|
'status_detail'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def format_ml_model(context, url, ml_model):
|
||||||
|
def transform(key, value):
|
||||||
|
if key not in _basic_keys:
|
||||||
|
return
|
||||||
|
# strip the key if it is not allowed by policy
|
||||||
|
policy_action = policies.ML_MODEL % ('get_one:%s' % key)
|
||||||
|
if not context.can(policy_action, fatal=False, might_not_exist=True):
|
||||||
|
return
|
||||||
|
if key == 'uuid':
|
||||||
|
yield ('uuid', value)
|
||||||
|
if url:
|
||||||
|
yield ('links', [link.make_link(
|
||||||
|
'self', url, 'ml_models', value),
|
||||||
|
link.make_link(
|
||||||
|
'bookmark', url,
|
||||||
|
'ml_models', value,
|
||||||
|
bookmark=True)])
|
||||||
|
else:
|
||||||
|
yield (key, value)
|
||||||
|
|
||||||
|
return dict(itertools.chain.from_iterable(
|
||||||
|
transform(k, v) for k, v in ml_model.items()))
|
||||||
145
gyan/api/controllers/versions.py
Normal file
145
gyan/api/controllers/versions.py
Normal file
@ -0,0 +1,145 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
|
||||||
|
from webob import exc
|
||||||
|
|
||||||
|
from gyan.common.i18n import _
|
||||||
|
|
||||||
|
# NOTE(tbh): v1.0 is reserved to indicate Ocata's API, but is not presently
|
||||||
|
# supported by the API service. All changes between Ocata and the
|
||||||
|
# point where we added microversioning are considered backwards-
|
||||||
|
# compatible, but are not specifically discoverable at this time.
|
||||||
|
#
|
||||||
|
# The v1.1 version indicates this "initial" version as being
|
||||||
|
# different from Ocata (v1.0), and includes the following changes:
|
||||||
|
#
|
||||||
|
# Add details of new api versions here:
|
||||||
|
|
||||||
|
#
|
||||||
|
# For each newly added microversion change, update the API version history
|
||||||
|
# string below with a one or two line description. Also update
|
||||||
|
# rest_api_version_history.rst for extra information on microversion.
|
||||||
|
REST_API_VERSION_HISTORY = """REST API Version History:
|
||||||
|
|
||||||
|
* 1.1 - Initial version
|
||||||
|
"""
|
||||||
|
|
||||||
|
BASE_VER = '1.1'
|
||||||
|
CURRENT_MAX_VER = '1.1'
|
||||||
|
|
||||||
|
|
||||||
|
class Version(object):
|
||||||
|
"""API Version object."""
|
||||||
|
|
||||||
|
string = 'OpenStack-API-Version'
|
||||||
|
"""HTTP Header string carrying the requested version"""
|
||||||
|
|
||||||
|
min_string = 'OpenStack-API-Minimum-Version'
|
||||||
|
"""HTTP response header"""
|
||||||
|
|
||||||
|
max_string = 'OpenStack-API-Maximum-Version'
|
||||||
|
"""HTTP response header"""
|
||||||
|
|
||||||
|
service_string = 'ml'
|
||||||
|
|
||||||
|
def __init__(self, headers, default_version, latest_version,
|
||||||
|
from_string=None):
|
||||||
|
"""Create an API Version object from the supplied headers.
|
||||||
|
|
||||||
|
:param headers: webob headers
|
||||||
|
:param default_version: version to use if not specified in headers
|
||||||
|
:param latest_version: version to use if latest is requested
|
||||||
|
:param from_string: create the version from string not headers
|
||||||
|
:raises: webob.HTTPNotAcceptable
|
||||||
|
"""
|
||||||
|
if from_string:
|
||||||
|
(self.major, self.minor) = tuple(int(i)
|
||||||
|
for i in from_string.split('.'))
|
||||||
|
|
||||||
|
else:
|
||||||
|
(self.major, self.minor) = Version.parse_headers(headers,
|
||||||
|
default_version,
|
||||||
|
latest_version)
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return '%s.%s' % (self.major, self.minor)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def parse_headers(headers, default_version, latest_version):
|
||||||
|
"""Determine the API version requested based on the headers supplied.
|
||||||
|
|
||||||
|
:param headers: webob headers
|
||||||
|
:param default_version: version to use if not specified in headers
|
||||||
|
:param latest_version: version to use if latest is requested
|
||||||
|
:returns: a tuple of (major, minor) version numbers
|
||||||
|
:raises: webob.HTTPNotAcceptable
|
||||||
|
"""
|
||||||
|
|
||||||
|
version_hdr = headers.get(Version.string, default_version)
|
||||||
|
|
||||||
|
try:
|
||||||
|
version_service, version_str = version_hdr.split()
|
||||||
|
except ValueError:
|
||||||
|
raise exc.HTTPNotAcceptable(_(
|
||||||
|
"Invalid service type for %s header") % Version.string)
|
||||||
|
|
||||||
|
if version_str.lower() == 'latest':
|
||||||
|
version_service, version_str = latest_version.split()
|
||||||
|
|
||||||
|
if version_service != Version.service_string:
|
||||||
|
raise exc.HTTPNotAcceptable(_(
|
||||||
|
"Invalid service type for %s header") % Version.string)
|
||||||
|
try:
|
||||||
|
version = tuple(int(i) for i in version_str.split('.'))
|
||||||
|
except ValueError:
|
||||||
|
version = ()
|
||||||
|
|
||||||
|
if len(version) != 2:
|
||||||
|
raise exc.HTTPNotAcceptable(_(
|
||||||
|
"Invalid value for %s header") % Version.string)
|
||||||
|
return version
|
||||||
|
|
||||||
|
def is_null(self):
|
||||||
|
return self.major == 0 and self.minor == 0
|
||||||
|
|
||||||
|
def matches(self, start_version, end_version):
|
||||||
|
if self.is_null():
|
||||||
|
raise ValueError
|
||||||
|
|
||||||
|
return start_version <= self <= end_version
|
||||||
|
|
||||||
|
def __lt__(self, other):
|
||||||
|
if self.major < other.major:
|
||||||
|
return True
|
||||||
|
if self.major == other.major and self.minor < other.minor:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def __gt__(self, other):
|
||||||
|
if self.major > other.major:
|
||||||
|
return True
|
||||||
|
if self.major == other.major and self.minor > other.minor:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def __eq__(self, other):
|
||||||
|
return self.major == other.major and self.minor == other.minor
|
||||||
|
|
||||||
|
def __le__(self, other):
|
||||||
|
return self < other or self == other
|
||||||
|
|
||||||
|
def __ne__(self, other):
|
||||||
|
return not self.__eq__(other)
|
||||||
|
|
||||||
|
def __ge__(self, other):
|
||||||
|
return self > other or self == other
|
||||||
114
gyan/api/hooks.py
Normal file
114
gyan/api/hooks.py
Normal file
@ -0,0 +1,114 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
|
||||||
|
from pecan import hooks
|
||||||
|
|
||||||
|
from gyan.common import context
|
||||||
|
from gyan.compute import api as compute_api
|
||||||
|
import gyan.conf
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
|
||||||
|
|
||||||
|
class ContextHook(hooks.PecanHook):
|
||||||
|
"""Configures a request context and attaches it to the request.
|
||||||
|
|
||||||
|
The following HTTP request headers are used:
|
||||||
|
|
||||||
|
X-User-Name:
|
||||||
|
Used for context.user_name.
|
||||||
|
|
||||||
|
X-User-Id:
|
||||||
|
Used for context.user_id.
|
||||||
|
|
||||||
|
X-Project-Name:
|
||||||
|
Used for context.project.
|
||||||
|
|
||||||
|
X-Project-Id:
|
||||||
|
Used for context.project_id.
|
||||||
|
|
||||||
|
X-Auth-Token:
|
||||||
|
Used for context.auth_token.
|
||||||
|
|
||||||
|
X-Roles:
|
||||||
|
Used for context.roles.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def before(self, state):
|
||||||
|
headers = state.request.headers
|
||||||
|
user_name = headers.get('X-User-Name')
|
||||||
|
user_id = headers.get('X-User-Id')
|
||||||
|
project = headers.get('X-Project-Name')
|
||||||
|
project_id = headers.get('X-Project-Id')
|
||||||
|
domain_id = headers.get('X-User-Domain-Id')
|
||||||
|
domain_name = headers.get('X-User-Domain-Name')
|
||||||
|
auth_token = headers.get('X-Auth-Token')
|
||||||
|
roles = headers.get('X-Roles', '').split(',')
|
||||||
|
auth_token_info = state.request.environ.get('keystone.token_info')
|
||||||
|
|
||||||
|
state.request.context = context.make_context(
|
||||||
|
auth_token=auth_token,
|
||||||
|
auth_token_info=auth_token_info,
|
||||||
|
user_name=user_name,
|
||||||
|
user_id=user_id,
|
||||||
|
project_name=project,
|
||||||
|
project_id=project_id,
|
||||||
|
domain_id=domain_id,
|
||||||
|
domain_name=domain_name,
|
||||||
|
roles=roles)
|
||||||
|
|
||||||
|
|
||||||
|
class RPCHook(hooks.PecanHook):
|
||||||
|
"""Attach the rpcapi object to the request so controllers can get to it."""
|
||||||
|
|
||||||
|
def before(self, state):
|
||||||
|
context = state.request.context
|
||||||
|
state.request.compute_api = compute_api.API(context)
|
||||||
|
|
||||||
|
|
||||||
|
class NoExceptionTracebackHook(hooks.PecanHook):
|
||||||
|
"""Workaround rpc.common: deserialize_remote_exception.
|
||||||
|
|
||||||
|
deserialize_remote_exception builds rpc exception traceback into error
|
||||||
|
message which is then sent to the client. Such behavior is a security
|
||||||
|
concern so this hook is aimed to cut-off traceback from the error message.
|
||||||
|
"""
|
||||||
|
# NOTE(tbh): 'after' hook used instead of 'on_error' because
|
||||||
|
# 'on_error' never fired for wsme+pecan pair. wsme @wsexpose decorator
|
||||||
|
# catches and handles all the errors, so 'on_error' dedicated for unhandled
|
||||||
|
# exceptions never fired.
|
||||||
|
def after(self, state):
|
||||||
|
# Omit empty body. Some errors may not have body at this level yet.
|
||||||
|
if not state.response.body:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Do nothing if there is no error.
|
||||||
|
if 200 <= state.response.status_int < 400:
|
||||||
|
return
|
||||||
|
|
||||||
|
json_body = state.response.json
|
||||||
|
# Do not remove traceback when server in debug mode (except 'Server'
|
||||||
|
# errors when 'debuginfo' will be used for traces).
|
||||||
|
if CONF.debug and json_body.get('faultcode') != 'Server':
|
||||||
|
return
|
||||||
|
|
||||||
|
title = json_body.get('title')
|
||||||
|
traceback_marker = 'Traceback (most recent call last):'
|
||||||
|
if title and (traceback_marker in title):
|
||||||
|
# Cut-off traceback.
|
||||||
|
title = title.split(traceback_marker, 1)[0]
|
||||||
|
# Remove trailing newlines and spaces if any.
|
||||||
|
json_body['title'] = title.rstrip()
|
||||||
|
# Replace the whole json. Cannot change original one beacause it's
|
||||||
|
# generated on the fly.
|
||||||
|
state.response.json = json_body
|
||||||
69
gyan/api/http_error.py
Normal file
69
gyan/api/http_error.py
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import six
|
||||||
|
|
||||||
|
from oslo_serialization import jsonutils as json
|
||||||
|
from webob import exc
|
||||||
|
|
||||||
|
|
||||||
|
class HTTPNotAcceptableAPIVersion(exc.HTTPNotAcceptable):
|
||||||
|
# subclass of :class:`~HTTPNotAcceptable`
|
||||||
|
#
|
||||||
|
# This indicates the resource identified by the request is only
|
||||||
|
# capable of generating response entities which have content
|
||||||
|
# characteristics not acceptable according to the accept headers
|
||||||
|
# sent in the request.
|
||||||
|
#
|
||||||
|
# code: 406, title: Not Acceptable
|
||||||
|
#
|
||||||
|
# differences from webob.exc.HTTPNotAcceptable:
|
||||||
|
#
|
||||||
|
# - additional max and min version parameters
|
||||||
|
# - additional error info for code, title, and links
|
||||||
|
code = 406
|
||||||
|
title = 'Not Acceptable'
|
||||||
|
max_version = ''
|
||||||
|
min_version = ''
|
||||||
|
|
||||||
|
def __init__(self, detail=None, headers=None, comment=None,
|
||||||
|
body_template=None, max_version='', min_version='', **kwargs):
|
||||||
|
|
||||||
|
super(HTTPNotAcceptableAPIVersion, self).__init__(
|
||||||
|
detail=detail, headers=headers, comment=comment,
|
||||||
|
body_template=body_template, **kwargs)
|
||||||
|
|
||||||
|
self.max_version = max_version
|
||||||
|
self.min_version = min_version
|
||||||
|
|
||||||
|
def __call__(self, environ, start_response):
|
||||||
|
for err_str in self.app_iter:
|
||||||
|
err = {}
|
||||||
|
try:
|
||||||
|
err = json.loads(err_str.decode('utf-8'))
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
links = {'rel': 'help', 'href': 'https://developer.openstack.org'
|
||||||
|
'/api-guide/compute/microversions.html'}
|
||||||
|
|
||||||
|
err['max_version'] = self.max_version
|
||||||
|
err['min_version'] = self.min_version
|
||||||
|
err['code'] = "gyan.microversion-unsupported"
|
||||||
|
err['links'] = [links]
|
||||||
|
err['title'] = "Requested microversion is unsupported"
|
||||||
|
|
||||||
|
self.app_iter = [six.b(json.dump_as_bytes(err))]
|
||||||
|
self.headers['Content-Length'] = str(len(self.app_iter[0]))
|
||||||
|
|
||||||
|
return super(HTTPNotAcceptableAPIVersion, self).__call__(
|
||||||
|
environ, start_response)
|
||||||
21
gyan/api/middleware/__init__.py
Normal file
21
gyan/api/middleware/__init__.py
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from gyan.api.middleware import auth_token
|
||||||
|
from gyan.api.middleware import parsable_error
|
||||||
|
|
||||||
|
|
||||||
|
AuthTokenMiddleware = auth_token.AuthTokenMiddleware
|
||||||
|
ParsableErrorMiddleware = parsable_error.ParsableErrorMiddleware
|
||||||
|
|
||||||
|
__all__ = ('AuthTokenMiddleware',
|
||||||
|
'ParsableErrorMiddleware')
|
||||||
71
gyan/api/middleware/auth_token.py
Normal file
71
gyan/api/middleware/auth_token.py
Normal file
@ -0,0 +1,71 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
from keystonemiddleware import auth_token
|
||||||
|
from oslo_log import log
|
||||||
|
|
||||||
|
from gyan.common import exception
|
||||||
|
from gyan.common.i18n import _
|
||||||
|
from gyan.common import utils
|
||||||
|
|
||||||
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class AuthTokenMiddleware(auth_token.AuthProtocol):
|
||||||
|
"""A wrapper on Keystone auth_token middleware.
|
||||||
|
|
||||||
|
Does not perform verification of authentication tokens
|
||||||
|
for public routes in the API.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, app, conf, public_api_routes=None):
|
||||||
|
if public_api_routes is None:
|
||||||
|
public_api_routes = []
|
||||||
|
route_pattern_tpl = '%s(\.json)?$'
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.public_api_routes = [re.compile(route_pattern_tpl % route_tpl)
|
||||||
|
for route_tpl in public_api_routes]
|
||||||
|
except re.error as e:
|
||||||
|
msg = _('Cannot compile public API routes: %s') % e
|
||||||
|
|
||||||
|
LOG.error(msg)
|
||||||
|
raise exception.ConfigInvalid(error_msg=msg)
|
||||||
|
|
||||||
|
super(AuthTokenMiddleware, self).__init__(app, conf)
|
||||||
|
|
||||||
|
def __call__(self, env, start_response):
|
||||||
|
path = utils.safe_rstrip(env.get('PATH_INFO'), '/')
|
||||||
|
|
||||||
|
# The information whether the API call is being performed against the
|
||||||
|
# public API is required for some other components. Saving it to the
|
||||||
|
# WSGI environment is reasonable thereby.
|
||||||
|
env['is_public_api'] = any([re.match(pattern, path)
|
||||||
|
for pattern in self.public_api_routes])
|
||||||
|
|
||||||
|
if env['is_public_api']:
|
||||||
|
return self._app(env, start_response)
|
||||||
|
|
||||||
|
return super(AuthTokenMiddleware, self).__call__(env, start_response)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def factory(cls, global_config, **local_conf):
|
||||||
|
public_routes = local_conf.get('acl_public_routes', '')
|
||||||
|
public_api_routes = [path.strip() for path in public_routes.split(',')]
|
||||||
|
|
||||||
|
def _factory(app):
|
||||||
|
return cls(app, global_config, public_api_routes=public_api_routes)
|
||||||
|
|
||||||
|
return _factory
|
||||||
99
gyan/api/middleware/parsable_error.py
Normal file
99
gyan/api/middleware/parsable_error.py
Normal file
@ -0,0 +1,99 @@
|
|||||||
|
# Copyright ? 2012 New Dream Network, LLC (DreamHost)
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
"""
|
||||||
|
Middleware to replace the plain text message body of an error
|
||||||
|
response with one formatted so the client can parse it.
|
||||||
|
|
||||||
|
Based on pecan.middleware.errordocument
|
||||||
|
"""
|
||||||
|
|
||||||
|
import six
|
||||||
|
|
||||||
|
from oslo_serialization import jsonutils as json
|
||||||
|
from gyan.common.i18n import _
|
||||||
|
|
||||||
|
|
||||||
|
class ParsableErrorMiddleware(object):
|
||||||
|
"""Replace error body with something the client can parse."""
|
||||||
|
|
||||||
|
def __init__(self, app):
|
||||||
|
self.app = app
|
||||||
|
|
||||||
|
def __call__(self, environ, start_response):
|
||||||
|
# Request for this state, modified by replace_start_response()
|
||||||
|
# and used when an error is being reported.
|
||||||
|
state = {}
|
||||||
|
|
||||||
|
def replacement_start_response(status, headers, exc_info=None):
|
||||||
|
"""Overrides the default response to make errors parsable."""
|
||||||
|
try:
|
||||||
|
status_code = int(status.split(' ')[0])
|
||||||
|
state['status_code'] = status_code
|
||||||
|
except (ValueError, TypeError): # pragma: nocover
|
||||||
|
raise Exception(_(
|
||||||
|
'ErrorDocumentMiddleware received an invalid '
|
||||||
|
'status %s') % status)
|
||||||
|
else:
|
||||||
|
if (state['status_code'] // 100) not in (2, 3):
|
||||||
|
# Remove some headers so we can replace them later
|
||||||
|
# when we have the full error message and can
|
||||||
|
# compute the length.
|
||||||
|
headers = [(h, v)
|
||||||
|
for (h, v) in headers
|
||||||
|
if h not in ('Content-Length', 'Content-Type')
|
||||||
|
]
|
||||||
|
# Save the headers in case we need to modify them.
|
||||||
|
state['headers'] = headers
|
||||||
|
return start_response(status, headers, exc_info)
|
||||||
|
|
||||||
|
app_iter = self.app(environ, replacement_start_response)
|
||||||
|
|
||||||
|
if (state['status_code'] // 100) not in (2, 3):
|
||||||
|
errs = []
|
||||||
|
for err_str in app_iter:
|
||||||
|
err = {}
|
||||||
|
try:
|
||||||
|
err = json.loads(err_str.decode('utf-8'))
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if 'title' in err and 'description' in err:
|
||||||
|
title = err['title']
|
||||||
|
desc = err['description']
|
||||||
|
else:
|
||||||
|
title = ''
|
||||||
|
desc = ''
|
||||||
|
|
||||||
|
error_code = err['faultstring'].lower() \
|
||||||
|
if 'faultstring' in err else ''
|
||||||
|
# 'ml-infra' is the service-name. The general form of the
|
||||||
|
# code is service-name.error-code.
|
||||||
|
code = '.'.join(['ml-infra', error_code])
|
||||||
|
|
||||||
|
errs.append({
|
||||||
|
'request_id': '',
|
||||||
|
'code': code,
|
||||||
|
'status': state['status_code'],
|
||||||
|
'title': title,
|
||||||
|
'detail': desc,
|
||||||
|
'links': []
|
||||||
|
})
|
||||||
|
|
||||||
|
body = [six.b(json.dumps({'errors': errs}))]
|
||||||
|
|
||||||
|
state['headers'].append(('Content-Type', 'application/json'))
|
||||||
|
state['headers'].append(('Content-Length', str(len(body[0]))))
|
||||||
|
else:
|
||||||
|
body = app_iter
|
||||||
|
return body
|
||||||
37
gyan/api/servicegroup.py
Normal file
37
gyan/api/servicegroup.py
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from oslo_utils import timeutils
|
||||||
|
|
||||||
|
import gyan.conf
|
||||||
|
from gyan import objects
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceGroup(object):
|
||||||
|
def __init__(self):
|
||||||
|
self.service_down_time = CONF.service_down_time
|
||||||
|
|
||||||
|
def service_is_up(self, member):
|
||||||
|
if not isinstance(member, objects.GyanService):
|
||||||
|
raise TypeError
|
||||||
|
if member.forced_down:
|
||||||
|
return False
|
||||||
|
|
||||||
|
last_heartbeat = (member.last_seen_up or
|
||||||
|
member.updated_at or member.created_at)
|
||||||
|
now = timeutils.utcnow()
|
||||||
|
elapsed = timeutils.delta_seconds(last_heartbeat, now)
|
||||||
|
is_up = abs(elapsed) <= self.service_down_time
|
||||||
|
return is_up
|
||||||
116
gyan/api/utils.py
Normal file
116
gyan/api/utils.py
Normal file
@ -0,0 +1,116 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import functools
|
||||||
|
from oslo_utils import uuidutils
|
||||||
|
import pecan
|
||||||
|
|
||||||
|
from gyan.api.controllers import versions
|
||||||
|
from gyan.common import exception
|
||||||
|
from gyan.common.i18n import _
|
||||||
|
import gyan.conf
|
||||||
|
from gyan import objects
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
|
||||||
|
|
||||||
|
def string_or_none(value):
|
||||||
|
if value in [None, 'None']:
|
||||||
|
return None
|
||||||
|
else:
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def validate_limit(limit):
|
||||||
|
try:
|
||||||
|
if limit is not None and int(limit) <= 0:
|
||||||
|
raise exception.InvalidValue(_("Limit must be positive integer"))
|
||||||
|
except ValueError:
|
||||||
|
raise exception.InvalidValue(_("Limit must be positive integer"))
|
||||||
|
|
||||||
|
if limit is not None:
|
||||||
|
return min(CONF.api.max_limit, int(limit))
|
||||||
|
else:
|
||||||
|
return CONF.api.max_limit
|
||||||
|
|
||||||
|
|
||||||
|
def validate_sort_dir(sort_dir):
|
||||||
|
if sort_dir not in ['asc', 'desc']:
|
||||||
|
raise exception.InvalidValue(_("Invalid sort direction: %s. "
|
||||||
|
"Acceptable values are "
|
||||||
|
"'asc' or 'desc'") % sort_dir)
|
||||||
|
return sort_dir
|
||||||
|
|
||||||
|
|
||||||
|
def get_resource(resource, resource_ident):
|
||||||
|
"""Get the resource from the uuid or logical name.
|
||||||
|
|
||||||
|
:param resource: the resource type.
|
||||||
|
:param resource_ident: the UUID or logical name of the resource.
|
||||||
|
|
||||||
|
:returns: The resource.
|
||||||
|
"""
|
||||||
|
resource = getattr(objects, resource)
|
||||||
|
context = pecan.request.context
|
||||||
|
if context.is_admin:
|
||||||
|
context.all_projects = True
|
||||||
|
if uuidutils.is_uuid_like(resource_ident):
|
||||||
|
return resource.get_by_uuid(context, resource_ident)
|
||||||
|
|
||||||
|
return resource.get_by_name(context, resource_ident)
|
||||||
|
|
||||||
|
|
||||||
|
def _do_enforce_content_types(pecan_req, valid_content_types):
|
||||||
|
"""Content type enforcement
|
||||||
|
|
||||||
|
Check to see that content type in the request is one of the valid
|
||||||
|
types passed in by our caller.
|
||||||
|
"""
|
||||||
|
if pecan_req.content_type not in valid_content_types:
|
||||||
|
m = (
|
||||||
|
"Unexpected content type: {type}. Expected content types "
|
||||||
|
"are: {expected}"
|
||||||
|
).format(
|
||||||
|
type=pecan_req.content_type.decode('utf-8'),
|
||||||
|
expected=valid_content_types
|
||||||
|
)
|
||||||
|
pecan.abort(415, m)
|
||||||
|
|
||||||
|
|
||||||
|
def enforce_content_types(valid_content_types):
|
||||||
|
"""Decorator handling content type enforcement on behalf of REST verbs."""
|
||||||
|
|
||||||
|
def content_types_decorator(fn):
|
||||||
|
|
||||||
|
@functools.wraps(fn)
|
||||||
|
def content_types_enforcer(inst, *args, **kwargs):
|
||||||
|
_do_enforce_content_types(pecan.request, valid_content_types)
|
||||||
|
return fn(inst, *args, **kwargs)
|
||||||
|
|
||||||
|
return content_types_enforcer
|
||||||
|
|
||||||
|
return content_types_decorator
|
||||||
|
|
||||||
|
|
||||||
|
def version_check(action, version):
|
||||||
|
"""Check whether the current version supports the operation.
|
||||||
|
|
||||||
|
:param action: Operations to be executed.
|
||||||
|
:param version: The minimum version required to perform the operation.
|
||||||
|
|
||||||
|
"""
|
||||||
|
req_version = pecan.request.version
|
||||||
|
min_version = versions.Version('', '', '', version)
|
||||||
|
if req_version < min_version:
|
||||||
|
raise exception.InvalidParamInVersion(param=action,
|
||||||
|
req_version=req_version,
|
||||||
|
min_version=min_version)
|
||||||
57
gyan/api/validation/__init__.py
Normal file
57
gyan/api/validation/__init__.py
Normal file
@ -0,0 +1,57 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import functools
|
||||||
|
|
||||||
|
from gyan.api.validation import validators
|
||||||
|
|
||||||
|
|
||||||
|
def validated(request_body_schema):
|
||||||
|
"""Register a schema to validate a resource reference.
|
||||||
|
|
||||||
|
Registered schema will be used for validating a request body just before
|
||||||
|
API method execution.
|
||||||
|
|
||||||
|
:param request_body_schema: a schema to validate the resource reference
|
||||||
|
"""
|
||||||
|
schema_validator = validators.SchemaValidator(request_body_schema,
|
||||||
|
is_body=True)
|
||||||
|
|
||||||
|
def add_validator(func):
|
||||||
|
@functools.wraps(func)
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
schema_validator.validate(kwargs)
|
||||||
|
return func(*args, **kwargs)
|
||||||
|
return wrapper
|
||||||
|
return add_validator
|
||||||
|
|
||||||
|
|
||||||
|
def validate_query_param(req, query_param_schema):
|
||||||
|
"""Register a schema to validate a resource reference.
|
||||||
|
|
||||||
|
Registered schema will be used for validating a request query params
|
||||||
|
just before API method execution.
|
||||||
|
|
||||||
|
:param req: the request object
|
||||||
|
:param query_param_schema: a schema to validate the resource reference
|
||||||
|
"""
|
||||||
|
|
||||||
|
schema_validator = validators.SchemaValidator(query_param_schema,
|
||||||
|
is_body=False)
|
||||||
|
|
||||||
|
def add_validator(func):
|
||||||
|
@functools.wraps(func)
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
schema_validator.validate(req.params.mixed())
|
||||||
|
return func(*args, **kwargs)
|
||||||
|
return wrapper
|
||||||
|
return add_validator
|
||||||
80
gyan/api/validation/validators.py
Normal file
80
gyan/api/validation/validators.py
Normal file
@ -0,0 +1,80 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import jsonschema
|
||||||
|
import six
|
||||||
|
|
||||||
|
from gyan.common import exception
|
||||||
|
from gyan.common.i18n import _
|
||||||
|
|
||||||
|
|
||||||
|
class SchemaValidator(object):
|
||||||
|
"""Resource reference validator class."""
|
||||||
|
|
||||||
|
validator_org = jsonschema.Draft4Validator
|
||||||
|
|
||||||
|
def __init__(self, schema, is_body=True):
|
||||||
|
self.is_body = is_body
|
||||||
|
validators = {
|
||||||
|
'minimum': self._validate_minimum,
|
||||||
|
'maximum': self._validate_maximum
|
||||||
|
}
|
||||||
|
validator_cls = jsonschema.validators.extend(self.validator_org,
|
||||||
|
validators)
|
||||||
|
fc = jsonschema.FormatChecker()
|
||||||
|
self.validator = validator_cls(schema, format_checker=fc)
|
||||||
|
|
||||||
|
def validate(self, *args, **kwargs):
|
||||||
|
try:
|
||||||
|
self.validator.validate(*args, **kwargs)
|
||||||
|
except jsonschema.ValidationError as ex:
|
||||||
|
if len(ex.path) > 0:
|
||||||
|
if self.is_body:
|
||||||
|
detail = _("Invalid input for field '%(path)s'."
|
||||||
|
"Value: '%(value)s'. %(message)s")
|
||||||
|
else:
|
||||||
|
detail = _("Invalid input for query parameters "
|
||||||
|
"'%(path)s'. Value: '%(value)s'. %(message)s")
|
||||||
|
detail = detail % {
|
||||||
|
'path': ex.path.pop(), 'value': ex.instance,
|
||||||
|
'message': six.text_type(ex)
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
detail = six.text_type(ex)
|
||||||
|
raise exception.SchemaValidationError(detail=detail)
|
||||||
|
|
||||||
|
def _number_from_str(self, instance):
|
||||||
|
if isinstance(instance, float) or isinstance(instance, int):
|
||||||
|
return instance
|
||||||
|
|
||||||
|
try:
|
||||||
|
value = int(instance)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
try:
|
||||||
|
value = float(instance)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
return None
|
||||||
|
return value
|
||||||
|
|
||||||
|
def _validate_minimum(self, validator, minimum, instance, schema):
|
||||||
|
instance = self._number_from_str(instance)
|
||||||
|
if instance is None:
|
||||||
|
return
|
||||||
|
return self.validator_org.VALIDATORS['minimum'](validator, minimum,
|
||||||
|
instance, schema)
|
||||||
|
|
||||||
|
def _validate_maximum(self, validator, maximum, instance, schema):
|
||||||
|
instance = self._number_from_str(instance)
|
||||||
|
if instance is None:
|
||||||
|
return
|
||||||
|
return self.validator_org.VALIDATORS['maximum'](validator, maximum,
|
||||||
|
instance, schema)
|
||||||
33
gyan/api/versioned_method.py
Normal file
33
gyan/api/versioned_method.py
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
|
||||||
|
class VersionedMethod(object):
|
||||||
|
|
||||||
|
def __init__(self, name, start_version, end_version, func):
|
||||||
|
"""Versioning information for a single method
|
||||||
|
|
||||||
|
@name: Name of the method
|
||||||
|
@start_version: Minimum acceptable version
|
||||||
|
@end_version: Maximum acceptable_version
|
||||||
|
@func: Method to call
|
||||||
|
|
||||||
|
Minimum and maximums are inclusive
|
||||||
|
"""
|
||||||
|
self.name = name
|
||||||
|
self.start_version = start_version
|
||||||
|
self.end_version = end_version
|
||||||
|
self.func = func
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return ("Version Method %s: min: %s, max: %s"
|
||||||
|
% (self.name, self.start_version, self.end_version))
|
||||||
36
gyan/api/wsgi.py
Normal file
36
gyan/api/wsgi.py
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from oslo_log import log
|
||||||
|
|
||||||
|
from gyan.api import app
|
||||||
|
from gyan.common import profiler
|
||||||
|
from gyan.common import service
|
||||||
|
import gyan.conf
|
||||||
|
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def init_application():
|
||||||
|
# Initialize the oslo configuration library and logging
|
||||||
|
service.prepare_service(sys.argv)
|
||||||
|
# NOTE:(tbh) change localhost to CONF.host
|
||||||
|
profiler.setup('gyan-api', "localhost")
|
||||||
|
|
||||||
|
LOG.debug("Configuration:")
|
||||||
|
CONF.log_opt_values(LOG, log.DEBUG)
|
||||||
|
|
||||||
|
return app.load_app()
|
||||||
17
gyan/cmd/__init__.py
Normal file
17
gyan/cmd/__init__.py
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
# NOTE(tbh): we monkey patch all eventlet services for easier tracking/debug
|
||||||
|
|
||||||
|
import eventlet
|
||||||
|
|
||||||
|
eventlet.monkey_patch()
|
||||||
45
gyan/cmd/api.py
Normal file
45
gyan/cmd/api.py
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""The gyan Service API."""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from gyan.common import profiler
|
||||||
|
from gyan.common import service as gyan_service
|
||||||
|
import gyan.conf
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
# Parse config file and command line options, then start logging
|
||||||
|
gyan_service.prepare_service(sys.argv)
|
||||||
|
|
||||||
|
# Enable object backporting via the conductor
|
||||||
|
# TODO(tbh): Uncomment after rpc services are implemented
|
||||||
|
# base.gyanObject.indirection_api = base.gyanObjectIndirectionAPI()
|
||||||
|
|
||||||
|
# Setup OSprofiler for WSGI service
|
||||||
|
profiler.setup('gyan-api', CONF.api.host_ip)
|
||||||
|
|
||||||
|
# Build and start the WSGI app
|
||||||
|
launcher = gyan_service.process_launcher()
|
||||||
|
server = gyan_service.WSGIService(
|
||||||
|
'gyan_api',
|
||||||
|
CONF.api.enable_ssl_api
|
||||||
|
)
|
||||||
|
launcher.launch_service(server, workers=server.workers)
|
||||||
|
launcher.wait()
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.exit(main())
|
||||||
47
gyan/cmd/compute.py
Normal file
47
gyan/cmd/compute.py
Normal file
@ -0,0 +1,47 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import os
|
||||||
|
import shlex
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from oslo_log import log as logging
|
||||||
|
from oslo_privsep import priv_context
|
||||||
|
from oslo_service import service
|
||||||
|
|
||||||
|
from gyan.common import rpc_service
|
||||||
|
from gyan.common import service as gyan_service
|
||||||
|
from gyan.common import utils
|
||||||
|
import gyan.conf
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
gyan_service.prepare_service(sys.argv)
|
||||||
|
|
||||||
|
LOG.info('Starting server in PID %s', os.getpid())
|
||||||
|
CONF.log_opt_values(LOG, logging.DEBUG)
|
||||||
|
|
||||||
|
CONF.import_opt('topic', 'gyan.conf.compute', group='compute')
|
||||||
|
CONF.import_opt('host', 'gyan.conf.compute', group='compute')
|
||||||
|
|
||||||
|
|
||||||
|
from gyan.compute import manager as compute_manager
|
||||||
|
endpoints = [
|
||||||
|
compute_manager.Manager(),
|
||||||
|
]
|
||||||
|
server = rpc_service.Service.create(CONF.compute.topic, CONF.compute.host,
|
||||||
|
endpoints, binary='gyan-compute')
|
||||||
|
launcher = service.launch(CONF, server, restart_method='mutate')
|
||||||
|
launcher.wait()
|
||||||
67
gyan/cmd/db_manage.py
Normal file
67
gyan/cmd/db_manage.py
Normal file
@ -0,0 +1,67 @@
|
|||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""Starter script for gyan-db-manage."""
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
|
||||||
|
from gyan.db import migration
|
||||||
|
|
||||||
|
|
||||||
|
CONF = cfg.CONF
|
||||||
|
|
||||||
|
|
||||||
|
def do_version():
|
||||||
|
print('Current DB revision is %s' % migration.version())
|
||||||
|
|
||||||
|
|
||||||
|
def do_upgrade():
|
||||||
|
migration.upgrade(CONF.command.revision)
|
||||||
|
|
||||||
|
|
||||||
|
def do_stamp():
|
||||||
|
migration.stamp(CONF.command.revision)
|
||||||
|
|
||||||
|
|
||||||
|
def do_revision():
|
||||||
|
migration.revision(message=CONF.command.message,
|
||||||
|
autogenerate=CONF.command.autogenerate)
|
||||||
|
|
||||||
|
|
||||||
|
def add_command_parsers(subparsers):
|
||||||
|
parser = subparsers.add_parser('version')
|
||||||
|
parser.set_defaults(func=do_version)
|
||||||
|
|
||||||
|
parser = subparsers.add_parser('upgrade')
|
||||||
|
parser.add_argument('revision', nargs='?')
|
||||||
|
parser.set_defaults(func=do_upgrade)
|
||||||
|
|
||||||
|
parser = subparsers.add_parser('stamp')
|
||||||
|
parser.add_argument('revision')
|
||||||
|
parser.set_defaults(func=do_stamp)
|
||||||
|
|
||||||
|
parser = subparsers.add_parser('revision')
|
||||||
|
parser.add_argument('-m', '--message')
|
||||||
|
parser.add_argument('--autogenerate', action='store_true')
|
||||||
|
parser.set_defaults(func=do_revision)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
command_opt = cfg.SubCommandOpt('command',
|
||||||
|
title='Command',
|
||||||
|
help='Available commands',
|
||||||
|
handler=add_command_parsers)
|
||||||
|
CONF.register_cli_opt(command_opt)
|
||||||
|
|
||||||
|
CONF(project='gyan')
|
||||||
|
CONF.command.func()
|
||||||
0
gyan/common/__init__.py
Normal file
0
gyan/common/__init__.py
Normal file
54
gyan/common/config.py
Normal file
54
gyan/common/config.py
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from oslo_middleware import cors
|
||||||
|
|
||||||
|
from gyan.common import rpc
|
||||||
|
import gyan.conf
|
||||||
|
from gyan import version
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args(argv, default_config_files=None):
|
||||||
|
rpc.set_defaults(control_exchange='gyan')
|
||||||
|
gyan.conf.CONF(argv[1:],
|
||||||
|
project='gyan',
|
||||||
|
version=version.version_info.release_string(),
|
||||||
|
default_config_files=default_config_files)
|
||||||
|
rpc.init(gyan.conf.CONF)
|
||||||
|
|
||||||
|
|
||||||
|
def set_config_defaults():
|
||||||
|
"""This method updates all configuration default values."""
|
||||||
|
set_cors_middleware_defaults()
|
||||||
|
|
||||||
|
|
||||||
|
def set_cors_middleware_defaults():
|
||||||
|
"""Update default configuration options for oslo.middleware."""
|
||||||
|
cors.set_defaults(
|
||||||
|
allow_headers=['X-Auth-Token',
|
||||||
|
'X-Identity-Status',
|
||||||
|
'X-Roles',
|
||||||
|
'X-Service-Catalog',
|
||||||
|
'X-User-Id',
|
||||||
|
'X-Project-Id',
|
||||||
|
'X-OpenStack-Request-ID',
|
||||||
|
'X-Server-Management-Url'],
|
||||||
|
expose_headers=['X-Auth-Token',
|
||||||
|
'X-Subject-Token',
|
||||||
|
'X-Service-Token',
|
||||||
|
'X-OpenStack-Request-ID',
|
||||||
|
'X-Server-Management-Url'],
|
||||||
|
allow_methods=['GET',
|
||||||
|
'PUT',
|
||||||
|
'POST',
|
||||||
|
'DELETE',
|
||||||
|
'PATCH'])
|
||||||
17
gyan/common/consts.py
Normal file
17
gyan/common/consts.py
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
ALLOCATED = 'allocated'
|
||||||
|
CREATED = 'created'
|
||||||
|
UNDEPLOYED = 'undeployed'
|
||||||
|
DEPLOYED = 'deployed'
|
||||||
180
gyan/common/context.py
Normal file
180
gyan/common/context.py
Normal file
@ -0,0 +1,180 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import functools
|
||||||
|
|
||||||
|
import copy
|
||||||
|
from oslo_context import context
|
||||||
|
from oslo_utils import timeutils
|
||||||
|
import six
|
||||||
|
|
||||||
|
from gyan.common import exception
|
||||||
|
from gyan.common import policy
|
||||||
|
|
||||||
|
|
||||||
|
class RequestContext(context.RequestContext):
|
||||||
|
"""Extends security contexts from the OpenStack common library."""
|
||||||
|
|
||||||
|
def __init__(self, auth_token=None, domain_id=None,
|
||||||
|
domain_name=None, user_name=None, user_id=None,
|
||||||
|
user_domain_name=None, user_domain_id=None,
|
||||||
|
project_name=None, project_id=None, roles=None,
|
||||||
|
is_admin=None, read_only=False, show_deleted=False,
|
||||||
|
request_id=None, trust_id=None, auth_token_info=None,
|
||||||
|
all_projects=False, password=None, timestamp=None, **kwargs):
|
||||||
|
"""Stores several additional request parameters:
|
||||||
|
|
||||||
|
:param domain_id: The ID of the domain.
|
||||||
|
:param domain_name: The name of the domain.
|
||||||
|
:param user_domain_id: The ID of the domain to
|
||||||
|
authenticate a user against.
|
||||||
|
:param user_domain_name: The name of the domain to
|
||||||
|
authenticate a user against.
|
||||||
|
|
||||||
|
"""
|
||||||
|
super(RequestContext, self).__init__(auth_token=auth_token,
|
||||||
|
user_id=user_name,
|
||||||
|
project_id=project_name,
|
||||||
|
is_admin=is_admin,
|
||||||
|
read_only=read_only,
|
||||||
|
show_deleted=show_deleted,
|
||||||
|
request_id=request_id,
|
||||||
|
roles=roles)
|
||||||
|
|
||||||
|
self.user_name = user_name
|
||||||
|
self.user_id = user_id
|
||||||
|
self.project_name = project_name
|
||||||
|
self.project_id = project_id
|
||||||
|
self.domain_id = domain_id
|
||||||
|
self.domain_name = domain_name
|
||||||
|
self.user_domain_id = user_domain_id
|
||||||
|
self.user_domain_name = user_domain_name
|
||||||
|
self.auth_token_info = auth_token_info
|
||||||
|
self.trust_id = trust_id
|
||||||
|
self.all_projects = all_projects
|
||||||
|
self.password = password
|
||||||
|
if is_admin is None:
|
||||||
|
self.is_admin = policy.check_is_admin(self)
|
||||||
|
else:
|
||||||
|
self.is_admin = is_admin
|
||||||
|
|
||||||
|
if not timestamp:
|
||||||
|
timestamp = timeutils.utcnow()
|
||||||
|
if isinstance(timestamp, six.string_types):
|
||||||
|
timestamp = timeutils.parse_strtime(timestamp)
|
||||||
|
self.timestamp = timestamp
|
||||||
|
|
||||||
|
def to_dict(self):
|
||||||
|
value = super(RequestContext, self).to_dict()
|
||||||
|
value.update({'auth_token': self.auth_token,
|
||||||
|
'domain_id': self.domain_id,
|
||||||
|
'domain_name': self.domain_name,
|
||||||
|
'user_domain_id': self.user_domain_id,
|
||||||
|
'user_domain_name': self.user_domain_name,
|
||||||
|
'user_name': self.user_name,
|
||||||
|
'user_id': self.user_id,
|
||||||
|
'project_name': self.project_name,
|
||||||
|
'project_id': self.project_id,
|
||||||
|
'is_admin': self.is_admin,
|
||||||
|
'read_only': self.read_only,
|
||||||
|
'roles': self.roles,
|
||||||
|
'show_deleted': self.show_deleted,
|
||||||
|
'request_id': self.request_id,
|
||||||
|
'trust_id': self.trust_id,
|
||||||
|
'auth_token_info': self.auth_token_info,
|
||||||
|
'password': self.password,
|
||||||
|
'all_projects': self.all_projects,
|
||||||
|
'timestamp': timeutils.strtime(self.timestamp) if
|
||||||
|
hasattr(self, 'timestamp') else None
|
||||||
|
})
|
||||||
|
return value
|
||||||
|
|
||||||
|
def to_policy_values(self):
|
||||||
|
policy = super(RequestContext, self).to_policy_values()
|
||||||
|
policy['is_admin'] = self.is_admin
|
||||||
|
return policy
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_dict(cls, values):
|
||||||
|
return cls(**values)
|
||||||
|
|
||||||
|
def elevated(self):
|
||||||
|
"""Return a version of this context with admin flag set."""
|
||||||
|
context = copy.copy(self)
|
||||||
|
# context.roles must be deepcopied to leave original roles
|
||||||
|
# without changes
|
||||||
|
context.roles = copy.deepcopy(self.roles)
|
||||||
|
context.is_admin = True
|
||||||
|
|
||||||
|
if 'admin' not in context.roles:
|
||||||
|
context.roles.append('admin')
|
||||||
|
|
||||||
|
return context
|
||||||
|
|
||||||
|
def can(self, action, target=None, fatal=True, might_not_exist=False):
|
||||||
|
"""Verifies that the given action is valid on the target in this context.
|
||||||
|
|
||||||
|
:param action: string representing the action to be checked.
|
||||||
|
:param target: dictionary representing the object of the action
|
||||||
|
for object creation this should be a dictionary representing the
|
||||||
|
location of the object e.g. ``{'project_id': context.project_id}``.
|
||||||
|
If None, then this default target will be considered:
|
||||||
|
{'project_id': self.project_id, 'user_id': self.user_id}
|
||||||
|
:param fatal: if False, will return False when an
|
||||||
|
exception.NotAuthorized occurs.
|
||||||
|
:param might_not_exist: If True the policy check is skipped (and the
|
||||||
|
function returns True) if the specified policy does not exist.
|
||||||
|
Defaults to false.
|
||||||
|
|
||||||
|
:raises gyan.common.exception.NotAuthorized: if verification fails and
|
||||||
|
fatal is True.
|
||||||
|
|
||||||
|
:return: returns a non-False value (not necessarily "True") if
|
||||||
|
authorized and False if not authorized and fatal is False.
|
||||||
|
"""
|
||||||
|
if target is None:
|
||||||
|
target = {'project_id': self.project_id,
|
||||||
|
'user_id': self.user_id}
|
||||||
|
|
||||||
|
try:
|
||||||
|
return policy.authorize(self, action, target,
|
||||||
|
might_not_exist=might_not_exist)
|
||||||
|
except exception.NotAuthorized:
|
||||||
|
if fatal:
|
||||||
|
raise
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def make_context(*args, **kwargs):
|
||||||
|
return RequestContext(*args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def get_admin_context(show_deleted=False, all_projects=False):
|
||||||
|
"""Create an administrator context.
|
||||||
|
|
||||||
|
:param show_deleted: if True, will show deleted items when query db
|
||||||
|
"""
|
||||||
|
context = RequestContext(user_id=None,
|
||||||
|
project=None,
|
||||||
|
is_admin=True,
|
||||||
|
show_deleted=show_deleted,
|
||||||
|
all_projects=all_projects)
|
||||||
|
return context
|
||||||
|
|
||||||
|
|
||||||
|
def set_context(func):
|
||||||
|
@functools.wraps(func)
|
||||||
|
def handler(self, ctx):
|
||||||
|
if ctx is None:
|
||||||
|
ctx = get_admin_context(all_projects=True)
|
||||||
|
func(self, ctx)
|
||||||
|
return handler
|
||||||
485
gyan/common/exception.py
Normal file
485
gyan/common/exception.py
Normal file
@ -0,0 +1,485 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
"""Gyan base exception handling.
|
||||||
|
|
||||||
|
Includes decorator for re-raising Gyan-type exceptions.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
import functools
|
||||||
|
import inspect
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from keystoneclient import exceptions as keystone_exceptions
|
||||||
|
from oslo_config import cfg
|
||||||
|
from oslo_log import log as logging
|
||||||
|
from oslo_utils import excutils
|
||||||
|
from oslo_utils import uuidutils
|
||||||
|
import pecan
|
||||||
|
import six
|
||||||
|
from webob import util as woutil
|
||||||
|
|
||||||
|
from gyan.common.i18n import _
|
||||||
|
import gyan.conf
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
|
||||||
|
try:
|
||||||
|
CONF.import_opt('fatal_exception_format_errors',
|
||||||
|
'oslo_versionedobjects.exception')
|
||||||
|
except cfg.NoSuchOptError as e:
|
||||||
|
# Note:work around for gyan run against master branch
|
||||||
|
# in devstack gate job, as gyan not branched yet
|
||||||
|
# versionobjects kilo/master different version can
|
||||||
|
# cause issue here. As it changed import group. So
|
||||||
|
# add here before branch to prevent gate failure.
|
||||||
|
# Bug: #1447873
|
||||||
|
CONF.import_opt('fatal_exception_format_errors',
|
||||||
|
'oslo_versionedobjects.exception',
|
||||||
|
group='oslo_versionedobjects')
|
||||||
|
|
||||||
|
|
||||||
|
def wrap_exception(notifier=None, event_type=None):
|
||||||
|
"""This decorator wraps a method to catch any exceptions.
|
||||||
|
|
||||||
|
It logs the exception as well as optionally sending
|
||||||
|
it to the notification system.
|
||||||
|
"""
|
||||||
|
def inner(f):
|
||||||
|
def wrapped(self, context, *args, **kwargs):
|
||||||
|
# Don't store self or context in the payload, it now seems to
|
||||||
|
# contain confidential information.
|
||||||
|
try:
|
||||||
|
return f(self, context, *args, **kwargs)
|
||||||
|
except Exception as e:
|
||||||
|
with excutils.save_and_reraise_exception():
|
||||||
|
if notifier:
|
||||||
|
call_dict = inspect.getcallargs(f, self, context,
|
||||||
|
*args, **kwargs)
|
||||||
|
payload = dict(exception=e,
|
||||||
|
private=dict(args=call_dict)
|
||||||
|
)
|
||||||
|
|
||||||
|
temp_type = event_type
|
||||||
|
if not temp_type:
|
||||||
|
# If f has multiple decorators, they must use
|
||||||
|
# functools.wraps to ensure the name is
|
||||||
|
# propagated.
|
||||||
|
temp_type = f.__name__
|
||||||
|
|
||||||
|
notifier.error(context, temp_type, payload)
|
||||||
|
|
||||||
|
return functools.wraps(f)(wrapped)
|
||||||
|
return inner
|
||||||
|
|
||||||
|
|
||||||
|
OBFUSCATED_MSG = _('Your request could not be handled '
|
||||||
|
'because of a problem in the server. '
|
||||||
|
'Error Correlation id is: %s')
|
||||||
|
|
||||||
|
|
||||||
|
def wrap_controller_exception(func, func_server_error, func_client_error):
|
||||||
|
"""This decorator wraps controllers methods to handle exceptions:
|
||||||
|
|
||||||
|
- if an unhandled Exception or a GyanException with an error code >=500
|
||||||
|
is catched, raise a http 5xx ClientSideError and correlates it with a log
|
||||||
|
message
|
||||||
|
|
||||||
|
- if a GyanException is catched and its error code is <500, raise a http
|
||||||
|
4xx and logs the excp in debug mode
|
||||||
|
|
||||||
|
"""
|
||||||
|
@functools.wraps(func)
|
||||||
|
def wrapped(*args, **kwargs):
|
||||||
|
try:
|
||||||
|
return func(*args, **kwargs)
|
||||||
|
except Exception as excp:
|
||||||
|
if isinstance(excp, GyanException):
|
||||||
|
http_error_code = excp.code
|
||||||
|
else:
|
||||||
|
http_error_code = 500
|
||||||
|
|
||||||
|
if http_error_code >= 500:
|
||||||
|
# log the error message with its associated
|
||||||
|
# correlation id
|
||||||
|
log_correlation_id = uuidutils.generate_uuid()
|
||||||
|
LOG.exception("%(correlation_id)s:%(excp)s",
|
||||||
|
{'correlation_id': log_correlation_id,
|
||||||
|
'excp': str(excp)})
|
||||||
|
# raise a client error with an obfuscated message
|
||||||
|
return func_server_error(log_correlation_id, http_error_code)
|
||||||
|
else:
|
||||||
|
# raise a client error the original message
|
||||||
|
LOG.debug(excp)
|
||||||
|
return func_client_error(excp, http_error_code)
|
||||||
|
return wrapped
|
||||||
|
|
||||||
|
|
||||||
|
def convert_excp_to_err_code(excp_name):
|
||||||
|
"""Convert Exception class name (CamelCase) to error-code (Snake-case)"""
|
||||||
|
words = re.findall(r'[A-Z]?[a-z]+|[A-Z]{2,}(?=[A-Z][a-z]|\d|\W|$)|\d+',
|
||||||
|
excp_name)
|
||||||
|
return '-'.join([str.lower(word) for word in words])
|
||||||
|
|
||||||
|
|
||||||
|
def wrap_pecan_controller_exception(func):
|
||||||
|
"""This decorator wraps pecan controllers to handle exceptions."""
|
||||||
|
def _func_server_error(log_correlation_id, status_code):
|
||||||
|
pecan.response.status = status_code
|
||||||
|
return {
|
||||||
|
'faultcode': 'Server',
|
||||||
|
'status_code': status_code,
|
||||||
|
'title': woutil.status_reasons[status_code],
|
||||||
|
'description': six.text_type(OBFUSCATED_MSG % log_correlation_id),
|
||||||
|
}
|
||||||
|
|
||||||
|
def _func_client_error(excp, status_code):
|
||||||
|
pecan.response.status = status_code
|
||||||
|
return {
|
||||||
|
'faultcode': 'Client',
|
||||||
|
'faultstring': convert_excp_to_err_code(excp.__class__.__name__),
|
||||||
|
'status_code': status_code,
|
||||||
|
'title': six.text_type(excp),
|
||||||
|
'description': six.text_type(excp),
|
||||||
|
}
|
||||||
|
|
||||||
|
return wrap_controller_exception(func,
|
||||||
|
_func_server_error,
|
||||||
|
_func_client_error)
|
||||||
|
|
||||||
|
|
||||||
|
def wrap_keystone_exception(func):
|
||||||
|
"""Wrap keystone exceptions and throw gyan specific exceptions."""
|
||||||
|
@functools.wraps(func)
|
||||||
|
def wrapped(*args, **kwargs):
|
||||||
|
try:
|
||||||
|
return func(*args, **kwargs)
|
||||||
|
except keystone_exceptions.AuthorizationFailure:
|
||||||
|
raise AuthorizationFailure(
|
||||||
|
client=func.__name__, message="reason: %s" % sys.exc_info()[1])
|
||||||
|
except keystone_exceptions.ClientException:
|
||||||
|
raise AuthorizationFailure(
|
||||||
|
client=func.__name__,
|
||||||
|
message="unexpected keystone client error occurred: %s"
|
||||||
|
% sys.exc_info()[1])
|
||||||
|
return wrapped
|
||||||
|
|
||||||
|
|
||||||
|
class GyanException(Exception):
|
||||||
|
"""Base gyan Exception
|
||||||
|
|
||||||
|
To correctly use this class, inherit from it and define
|
||||||
|
a 'message' property. That message will get printf'd
|
||||||
|
with the keyword arguments provided to the constructor.
|
||||||
|
|
||||||
|
"""
|
||||||
|
message = _("An unknown exception occurred.")
|
||||||
|
code = 500
|
||||||
|
|
||||||
|
def __init__(self, message=None, **kwargs):
|
||||||
|
self.kwargs = kwargs
|
||||||
|
|
||||||
|
if 'code' not in self.kwargs and hasattr(self, 'code'):
|
||||||
|
self.kwargs['code'] = self.code
|
||||||
|
|
||||||
|
if message:
|
||||||
|
self.message = message
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.message = str(self.message) % kwargs
|
||||||
|
except KeyError:
|
||||||
|
# kwargs doesn't match a variable in the message
|
||||||
|
# log the issue and the kwargs
|
||||||
|
LOG.exception('Exception in string format operation, '
|
||||||
|
'kwargs: %s', kwargs)
|
||||||
|
try:
|
||||||
|
ferr = CONF.fatal_exception_format_errors
|
||||||
|
except cfg.NoSuchOptError:
|
||||||
|
ferr = CONF.oslo_versionedobjects.fatal_exception_format_errors
|
||||||
|
if ferr:
|
||||||
|
raise
|
||||||
|
|
||||||
|
super(GyanException, self).__init__(self.message)
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
if six.PY3:
|
||||||
|
return self.message
|
||||||
|
return self.message.encode('utf-8')
|
||||||
|
|
||||||
|
def __unicode__(self):
|
||||||
|
return self.message
|
||||||
|
|
||||||
|
def format_message(self):
|
||||||
|
if self.__class__.__name__.endswith('_Remote'):
|
||||||
|
return self.args[0]
|
||||||
|
else:
|
||||||
|
return six.text_type(self)
|
||||||
|
|
||||||
|
|
||||||
|
class ObjectNotFound(GyanException):
|
||||||
|
message = _("The %(name)s %(id)s could not be found.")
|
||||||
|
|
||||||
|
|
||||||
|
class ObjectNotUnique(GyanException):
|
||||||
|
message = _("The %(name)s already exists.")
|
||||||
|
|
||||||
|
|
||||||
|
class ObjectActionError(GyanException):
|
||||||
|
message = _('Object action %(action)s failed because: %(reason)s')
|
||||||
|
|
||||||
|
|
||||||
|
class ResourceNotFound(ObjectNotFound):
|
||||||
|
message = _("The %(name)s resource %(id)s could not be found.")
|
||||||
|
code = 404
|
||||||
|
|
||||||
|
|
||||||
|
class ResourceExists(ObjectNotUnique):
|
||||||
|
message = _("The %(name)s resource already exists.")
|
||||||
|
code = 409
|
||||||
|
|
||||||
|
|
||||||
|
class AuthorizationFailure(GyanException):
|
||||||
|
message = _("%(client)s connection failed. %(message)s")
|
||||||
|
|
||||||
|
|
||||||
|
class UnsupportedObjectError(GyanException):
|
||||||
|
message = _('Unsupported object type %(objtype)s')
|
||||||
|
|
||||||
|
|
||||||
|
class IncompatibleObjectVersion(GyanException):
|
||||||
|
message = _('Version %(objver)s of %(objname)s is not supported')
|
||||||
|
|
||||||
|
|
||||||
|
class OrphanedObjectError(GyanException):
|
||||||
|
message = _('Cannot call %(method)s on orphaned %(objtype)s object')
|
||||||
|
|
||||||
|
|
||||||
|
class Invalid(GyanException):
|
||||||
|
message = _("Unacceptable parameters.")
|
||||||
|
code = 400
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidValue(Invalid):
|
||||||
|
message = _("Received value '%(value)s' is invalid for type %(type)s.")
|
||||||
|
|
||||||
|
|
||||||
|
class ValidationError(Invalid):
|
||||||
|
message = "%(detail)s"
|
||||||
|
|
||||||
|
|
||||||
|
class SchemaValidationError(ValidationError):
|
||||||
|
message = "%(detail)s"
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidUUID(Invalid):
|
||||||
|
message = _("Expected a uuid but received %(uuid)s.")
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidName(Invalid):
|
||||||
|
message = _("Expected a name but received %(uuid)s.")
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidDiscoveryURL(Invalid):
|
||||||
|
message = _("Received invalid discovery URL '%(discovery_url)s' for "
|
||||||
|
"discovery endpoint '%(discovery_endpoint)s'.")
|
||||||
|
|
||||||
|
|
||||||
|
class GetDiscoveryUrlFailed(GyanException):
|
||||||
|
message = _("Failed to get discovery url from '%(discovery_endpoint)s'.")
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidUuidOrName(Invalid):
|
||||||
|
message = _("Expected a name or uuid but received %(uuid)s.")
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidIdentity(Invalid):
|
||||||
|
message = _("Expected an uuid or int but received %(identity)s.")
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidCsr(Invalid):
|
||||||
|
message = _("Received invalid csr %(csr)s.")
|
||||||
|
|
||||||
|
|
||||||
|
class HTTPNotFound(ResourceNotFound):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class Conflict(GyanException):
|
||||||
|
message = _('Conflict.')
|
||||||
|
code = 409
|
||||||
|
|
||||||
|
|
||||||
|
class ConflictOptions(Conflict):
|
||||||
|
message = _('Conflicting options.')
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidState(Conflict):
|
||||||
|
message = _("Invalid resource state.")
|
||||||
|
|
||||||
|
|
||||||
|
# Cannot be templated as the error syntax varies.
|
||||||
|
# msg needs to be constructed when raised.
|
||||||
|
class InvalidParameterValue(Invalid):
|
||||||
|
message = _("%(err)s")
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidParamInVersion(Invalid):
|
||||||
|
message = _('Invalid param %(param)s because current request '
|
||||||
|
'version is %(req_version)s. %(param)s is only '
|
||||||
|
'supported from version %(min_version)s')
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidQuotaValue(Invalid):
|
||||||
|
message = _("Change would make usage less than 0 for the following "
|
||||||
|
"resources: %(unders)s")
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidQuotaMethodUsage(Invalid):
|
||||||
|
message = _("Wrong quota method %(method)s used on resource %(res)s")
|
||||||
|
|
||||||
|
|
||||||
|
class PatchError(Invalid):
|
||||||
|
message = _("Couldn't apply patch '%(patch)s'. Reason: %(reason)s")
|
||||||
|
|
||||||
|
|
||||||
|
class NotAuthorized(GyanException):
|
||||||
|
message = _("Not authorized.")
|
||||||
|
code = 403
|
||||||
|
|
||||||
|
|
||||||
|
class MLModelAlreadyExists(GyanException):
|
||||||
|
message = _("A ML Model with %(field)s %(value)s already exists.")
|
||||||
|
|
||||||
|
|
||||||
|
class MLModelNotFound(GyanException):
|
||||||
|
message = _("ML Model %(ml_model)s could not be found.")
|
||||||
|
|
||||||
|
class ConfigInvalid(GyanException):
|
||||||
|
message = _("Invalid configuration file. %(error_msg)s")
|
||||||
|
|
||||||
|
|
||||||
|
class PolicyNotAuthorized(NotAuthorized):
|
||||||
|
message = _("Policy doesn't allow %(action)s to be performed.")
|
||||||
|
|
||||||
|
|
||||||
|
class ComputeHostNotFound(HTTPNotFound):
|
||||||
|
message = _("Compute host %(compute_host)s could not be found.")
|
||||||
|
|
||||||
|
|
||||||
|
class GyanServiceNotFound(HTTPNotFound):
|
||||||
|
message = _("Gyan service %(binary)s on host %(host)s could not be found.")
|
||||||
|
|
||||||
|
|
||||||
|
class ResourceProviderNotFound(HTTPNotFound):
|
||||||
|
message = _("Resource provider %(resource_provider)s could not be found.")
|
||||||
|
|
||||||
|
|
||||||
|
class ResourceClassNotFound(HTTPNotFound):
|
||||||
|
message = _("Resource class %(resource_class)s could not be found.")
|
||||||
|
|
||||||
|
|
||||||
|
class ComputeHostAlreadyExists(ResourceExists):
|
||||||
|
message = _("A compute host with %(field)s %(value)s already exists.")
|
||||||
|
|
||||||
|
|
||||||
|
class GyanServiceAlreadyExists(ResourceExists):
|
||||||
|
message = _("Service %(binary)s on host %(host)s already exists.")
|
||||||
|
|
||||||
|
|
||||||
|
class ResourceProviderAlreadyExists(ResourceExists):
|
||||||
|
message = _("A resource provider with %(field)s %(value)s already exists.")
|
||||||
|
|
||||||
|
|
||||||
|
class ResourceClassAlreadyExists(ResourceExists):
|
||||||
|
message = _("A resource class with %(field)s %(value)s already exists.")
|
||||||
|
|
||||||
|
|
||||||
|
class UniqueConstraintViolated(ResourceExists):
|
||||||
|
message = _("A resource with %(fields)s violates unique constraint.")
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidStateException(GyanException):
|
||||||
|
message = _("Cannot %(action)s ml model %(id)s in %(actual_state)s state")
|
||||||
|
code = 409
|
||||||
|
|
||||||
|
|
||||||
|
class ServerInError(GyanException):
|
||||||
|
message = _('Went to status %(resource_status)s due to '
|
||||||
|
'"%(status_reason)s"')
|
||||||
|
|
||||||
|
|
||||||
|
class ServerUnknownStatus(GyanException):
|
||||||
|
message = _('%(result)s - Unknown status %(resource_status)s due to '
|
||||||
|
'"%(status_reason)s"')
|
||||||
|
|
||||||
|
|
||||||
|
class EntityNotFound(GyanException):
|
||||||
|
message = _("The %(entity)s (%(name)s) could not be found.")
|
||||||
|
|
||||||
|
|
||||||
|
class CommandError(GyanException):
|
||||||
|
message = _("The command: %(cmd)s failed on the system, due to %(error)s")
|
||||||
|
|
||||||
|
|
||||||
|
class NoValidHost(GyanException):
|
||||||
|
message = _("No valid host was found. %(reason)s")
|
||||||
|
|
||||||
|
|
||||||
|
class NotFound(GyanException):
|
||||||
|
message = _("Resource could not be found.")
|
||||||
|
code = 404
|
||||||
|
|
||||||
|
|
||||||
|
class SchedulerHostFilterNotFound(NotFound):
|
||||||
|
message = _("Scheduler Host Filter %(filter_name)s could not be found.")
|
||||||
|
|
||||||
|
|
||||||
|
class ClassNotFound(NotFound):
|
||||||
|
message = _("Class %(class_name)s could not be found: %(exception)s")
|
||||||
|
|
||||||
|
|
||||||
|
class ApiVersionsIntersect(GyanException):
|
||||||
|
message = _("Version of %(name)s %(min_ver)s %(max_ver)s intersects "
|
||||||
|
"with another versions.")
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectionFailed(GyanException):
|
||||||
|
message = _("Failed to connect to remote host")
|
||||||
|
|
||||||
|
|
||||||
|
class SocketException(GyanException):
|
||||||
|
message = _("Socket exceptions")
|
||||||
|
|
||||||
|
|
||||||
|
class ResourcesUnavailable(GyanException):
|
||||||
|
message = _("Insufficient compute resources: %(reason)s.")
|
||||||
|
|
||||||
|
|
||||||
|
class FileNotFound(GyanException):
|
||||||
|
message = _("The expected file not exist")
|
||||||
|
|
||||||
|
|
||||||
|
class FailedParseStringToJson(GyanException):
|
||||||
|
message = _("Failed parse string to json: %(reason)s.")
|
||||||
|
|
||||||
|
|
||||||
|
class ServerNotUsable(GyanException):
|
||||||
|
message = _("gyan server not usable")
|
||||||
|
code = 404
|
||||||
|
|
||||||
|
|
||||||
|
class Base64Exception(Invalid):
|
||||||
|
msg_fmt = _("Invalid Base 64 file data")
|
||||||
24
gyan/common/i18n.py
Normal file
24
gyan/common/i18n.py
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
# It's based on oslo.i18n usage in OpenStack Keystone project and
|
||||||
|
# recommendations from
|
||||||
|
# https://docs.openstack.org/oslo.i18n/latest/user/usage.html
|
||||||
|
|
||||||
|
import oslo_i18n
|
||||||
|
|
||||||
|
|
||||||
|
_translators = oslo_i18n.TranslatorFactory(domain='gyan')
|
||||||
|
|
||||||
|
# The primary translation function using the well-known name "_"
|
||||||
|
_ = _translators.primary
|
||||||
89
gyan/common/keystone.py
Normal file
89
gyan/common/keystone.py
Normal file
@ -0,0 +1,89 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from keystoneauth1.access import access as ka_access
|
||||||
|
from keystoneauth1.identity import access as ka_access_plugin
|
||||||
|
from keystoneauth1.identity import v3 as ka_v3
|
||||||
|
from keystoneauth1 import loading as ka_loading
|
||||||
|
from keystoneclient.v3 import client as kc_v3
|
||||||
|
from oslo_log import log as logging
|
||||||
|
|
||||||
|
from gyan.common import exception
|
||||||
|
import gyan.conf
|
||||||
|
from gyan.conf import keystone as ksconf
|
||||||
|
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class KeystoneClientV3(object):
|
||||||
|
"""Keystone client wrapper so we can encapsulate logic in one place."""
|
||||||
|
|
||||||
|
def __init__(self, context):
|
||||||
|
self.context = context
|
||||||
|
self._client = None
|
||||||
|
self._session = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def auth_url(self):
|
||||||
|
# FIXME(pauloewerton): auth_url should be retrieved from keystone_auth
|
||||||
|
# section by default
|
||||||
|
url = CONF[ksconf.CFG_LEGACY_GROUP].www_authenticate_uri or \
|
||||||
|
CONF[ksconf.CFG_LEGACY_GROUP].auth_uri
|
||||||
|
return url.replace('v2.0', 'v3')
|
||||||
|
|
||||||
|
@property
|
||||||
|
def auth_token(self):
|
||||||
|
return self.session.get_token()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def session(self):
|
||||||
|
if self._session:
|
||||||
|
return self._session
|
||||||
|
auth = self._get_auth()
|
||||||
|
session = self._get_session(auth)
|
||||||
|
self._session = session
|
||||||
|
return session
|
||||||
|
|
||||||
|
def _get_session(self, auth):
|
||||||
|
session = ka_loading.load_session_from_conf_options(
|
||||||
|
CONF, ksconf.CFG_GROUP, auth=auth)
|
||||||
|
return session
|
||||||
|
|
||||||
|
def _get_auth(self):
|
||||||
|
if self.context.auth_token_info:
|
||||||
|
access_info = ka_access.create(body=self.context.auth_token_info,
|
||||||
|
auth_token=self.context.auth_token)
|
||||||
|
auth = ka_access_plugin.AccessInfoPlugin(access_info)
|
||||||
|
elif self.context.auth_token:
|
||||||
|
auth = ka_v3.Token(auth_url=self.auth_url,
|
||||||
|
token=self.context.auth_token)
|
||||||
|
elif self.context.is_admin:
|
||||||
|
auth = ka_loading.load_auth_from_conf_options(CONF,
|
||||||
|
ksconf.CFG_GROUP)
|
||||||
|
else:
|
||||||
|
msg = ('Keystone API connection failed: no password, '
|
||||||
|
'trust_id or token found.')
|
||||||
|
LOG.error(msg)
|
||||||
|
raise exception.AuthorizationFailure(client='keystone',
|
||||||
|
message='reason %s' % msg)
|
||||||
|
|
||||||
|
return auth
|
||||||
|
|
||||||
|
@property
|
||||||
|
def client(self):
|
||||||
|
if self._client:
|
||||||
|
return self._client
|
||||||
|
client = kc_v3.Client(session=self.session)
|
||||||
|
self._client = client
|
||||||
|
return client
|
||||||
47
gyan/common/paths.py
Normal file
47
gyan/common/paths.py
Normal file
@ -0,0 +1,47 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
import gyan.conf
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
|
||||||
|
|
||||||
|
def basedir_def(*args):
|
||||||
|
"""Return an uninterpolated path relative to $pybasedir."""
|
||||||
|
return os.path.join('$pybasedir', *args)
|
||||||
|
|
||||||
|
|
||||||
|
def bindir_def(*args):
|
||||||
|
"""Return an uninterpolated path relative to $bindir."""
|
||||||
|
return os.path.join('$bindir', *args)
|
||||||
|
|
||||||
|
|
||||||
|
def state_path_def(*args):
|
||||||
|
"""Return an uninterpolated path relative to $state_path."""
|
||||||
|
return os.path.join('$state_path', *args)
|
||||||
|
|
||||||
|
|
||||||
|
def basedir_rel(*args):
|
||||||
|
"""Return a path relative to $pybasedir."""
|
||||||
|
return os.path.join(CONF.pybasedir, *args)
|
||||||
|
|
||||||
|
|
||||||
|
def bindir_rel(*args):
|
||||||
|
"""Return a path relative to $bindir."""
|
||||||
|
return os.path.join(CONF.bindir, *args)
|
||||||
|
|
||||||
|
|
||||||
|
def state_path_rel(*args):
|
||||||
|
"""Return a path relative to $state_path."""
|
||||||
|
return os.path.join(CONF.state_path, *args)
|
||||||
24
gyan/common/policies/__init__.py
Normal file
24
gyan/common/policies/__init__.py
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import itertools
|
||||||
|
|
||||||
|
from gyan.common.policies import host
|
||||||
|
from gyan.common.policies import base
|
||||||
|
from gyan.common.policies import ml_model
|
||||||
|
|
||||||
|
def list_rules():
|
||||||
|
return itertools.chain(
|
||||||
|
base.list_rules(),
|
||||||
|
host.list_rules(),
|
||||||
|
ml_model.list_rules()
|
||||||
|
)
|
||||||
41
gyan/common/policies/base.py
Normal file
41
gyan/common/policies/base.py
Normal file
@ -0,0 +1,41 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from oslo_policy import policy
|
||||||
|
|
||||||
|
ROLE_ADMIN = 'role:admin'
|
||||||
|
RULE_ADMIN_OR_OWNER = 'is_admin:True or project_id:%(project_id)s'
|
||||||
|
RULE_ADMIN_API = 'rule:context_is_admin'
|
||||||
|
RULE_DENY_EVERYBODY = 'rule:deny_everybody'
|
||||||
|
|
||||||
|
rules = [
|
||||||
|
policy.RuleDefault(
|
||||||
|
name='context_is_admin',
|
||||||
|
check_str=ROLE_ADMIN
|
||||||
|
),
|
||||||
|
policy.RuleDefault(
|
||||||
|
name='admin_or_owner',
|
||||||
|
check_str=RULE_ADMIN_OR_OWNER
|
||||||
|
),
|
||||||
|
policy.RuleDefault(
|
||||||
|
name='admin_api',
|
||||||
|
check_str=RULE_ADMIN_API
|
||||||
|
),
|
||||||
|
policy.RuleDefault(
|
||||||
|
name="deny_everybody",
|
||||||
|
check_str="!",
|
||||||
|
description="Default rule for deny everybody."),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def list_rules():
|
||||||
|
return rules
|
||||||
46
gyan/common/policies/host.py
Normal file
46
gyan/common/policies/host.py
Normal file
@ -0,0 +1,46 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from oslo_policy import policy
|
||||||
|
|
||||||
|
from gyan.common.policies import base
|
||||||
|
|
||||||
|
HOST = 'host:%s'
|
||||||
|
|
||||||
|
rules = [
|
||||||
|
policy.DocumentedRuleDefault(
|
||||||
|
name=HOST % 'get_all',
|
||||||
|
check_str=base.RULE_ADMIN_API,
|
||||||
|
description='List all compute hosts.',
|
||||||
|
operations=[
|
||||||
|
{
|
||||||
|
'path': '/v1/hosts',
|
||||||
|
'method': 'GET'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
),
|
||||||
|
policy.DocumentedRuleDefault(
|
||||||
|
name=HOST % 'get',
|
||||||
|
check_str=base.RULE_ADMIN_API,
|
||||||
|
description='Show the details of a specific compute host.',
|
||||||
|
operations=[
|
||||||
|
{
|
||||||
|
'path': '/v1/hosts/{host_ident}',
|
||||||
|
'method': 'GET'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def list_rules():
|
||||||
|
return rules
|
||||||
123
gyan/common/policies/ml_model.py
Normal file
123
gyan/common/policies/ml_model.py
Normal file
@ -0,0 +1,123 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from oslo_policy import policy
|
||||||
|
|
||||||
|
from gyan.common.policies import base
|
||||||
|
|
||||||
|
ML_MODEL = 'ml_model:%s'
|
||||||
|
|
||||||
|
rules = [
|
||||||
|
policy.DocumentedRuleDefault(
|
||||||
|
name=ML_MODEL % 'create',
|
||||||
|
check_str=base.RULE_ADMIN_OR_OWNER,
|
||||||
|
description='Create a new ML Model.',
|
||||||
|
operations=[
|
||||||
|
{
|
||||||
|
'path': '/v1/ml_models',
|
||||||
|
'method': 'POST'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
),
|
||||||
|
policy.DocumentedRuleDefault(
|
||||||
|
name=ML_MODEL % 'delete',
|
||||||
|
check_str=base.RULE_ADMIN_OR_OWNER,
|
||||||
|
description='Delete a ML Model.',
|
||||||
|
operations=[
|
||||||
|
{
|
||||||
|
'path': '/v1/ml_models/{ml_model_ident}',
|
||||||
|
'method': 'DELETE'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
),
|
||||||
|
policy.DocumentedRuleDefault(
|
||||||
|
name=ML_MODEL % 'delete_all_projects',
|
||||||
|
check_str=base.RULE_ADMIN_API,
|
||||||
|
description='Delete a ml models from all projects.',
|
||||||
|
operations=[
|
||||||
|
{
|
||||||
|
'path': '/v1/ml_models/{ml_model_ident}',
|
||||||
|
'method': 'DELETE'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
),
|
||||||
|
policy.DocumentedRuleDefault(
|
||||||
|
name=ML_MODEL % 'delete_force',
|
||||||
|
check_str=base.RULE_ADMIN_API,
|
||||||
|
description='Forcibly delete a ML model.',
|
||||||
|
operations=[
|
||||||
|
{
|
||||||
|
'path': '/v1/ml_models/{ml_model_ident}',
|
||||||
|
'method': 'DELETE'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
),
|
||||||
|
policy.DocumentedRuleDefault(
|
||||||
|
name=ML_MODEL % 'get_one',
|
||||||
|
check_str=base.RULE_ADMIN_OR_OWNER,
|
||||||
|
description='Retrieve the details of a specific ml model.',
|
||||||
|
operations=[
|
||||||
|
{
|
||||||
|
'path': '/v1/ml_models/{ml_model_ident}',
|
||||||
|
'method': 'GET'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
),
|
||||||
|
policy.DocumentedRuleDefault(
|
||||||
|
name=ML_MODEL % 'get_all',
|
||||||
|
check_str=base.RULE_ADMIN_OR_OWNER,
|
||||||
|
description='Retrieve the details of all ml models.',
|
||||||
|
operations=[
|
||||||
|
{
|
||||||
|
'path': '/v1/ml_models',
|
||||||
|
'method': 'GET'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
),
|
||||||
|
policy.DocumentedRuleDefault(
|
||||||
|
name=ML_MODEL % 'get_all_all_projects',
|
||||||
|
check_str=base.RULE_ADMIN_API,
|
||||||
|
description='Retrieve the details of all ml models across projects.',
|
||||||
|
operations=[
|
||||||
|
{
|
||||||
|
'path': '/v1/ml_models',
|
||||||
|
'method': 'GET'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
),
|
||||||
|
policy.DocumentedRuleDefault(
|
||||||
|
name=ML_MODEL % 'update',
|
||||||
|
check_str=base.RULE_ADMIN_OR_OWNER,
|
||||||
|
description='Update a ML Model.',
|
||||||
|
operations=[
|
||||||
|
{
|
||||||
|
'path': '/v1/ml_models/{ml_model_ident}',
|
||||||
|
'method': 'PATCH'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
),
|
||||||
|
policy.DocumentedRuleDefault(
|
||||||
|
name=ML_MODEL % 'upload',
|
||||||
|
check_str=base.RULE_ADMIN_OR_OWNER,
|
||||||
|
description='Upload the trained ML Model',
|
||||||
|
operations=[
|
||||||
|
{
|
||||||
|
'path': '/v1/ml_models/{ml_model_ident}/upload',
|
||||||
|
'method': 'POST'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def list_rules():
|
||||||
|
return rules
|
||||||
155
gyan/common/policy.py
Normal file
155
gyan/common/policy.py
Normal file
@ -0,0 +1,155 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""Policy Engine For Gyan."""
|
||||||
|
|
||||||
|
from oslo_log import log as logging
|
||||||
|
from oslo_policy import policy
|
||||||
|
from oslo_utils import excutils
|
||||||
|
|
||||||
|
from gyan.common import exception
|
||||||
|
from gyan.common import policies
|
||||||
|
import gyan.conf
|
||||||
|
|
||||||
|
_ENFORCER = None
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
# we can get a policy enforcer by this init.
|
||||||
|
# oslo policy support change policy rule dynamically.
|
||||||
|
# at present, policy.enforce will reload the policy rules when it checks
|
||||||
|
# the policy files have been touched.
|
||||||
|
def init(policy_file=None, rules=None,
|
||||||
|
default_rule=None, use_conf=True, overwrite=True):
|
||||||
|
"""Init an Enforcer class.
|
||||||
|
|
||||||
|
:param policy_file: Custom policy file to use, if none is
|
||||||
|
specified, ``conf.policy_file`` will be
|
||||||
|
used.
|
||||||
|
:param rules: Default dictionary / Rules to use. It will be
|
||||||
|
considered just in the first instantiation. If
|
||||||
|
:meth:`load_rules` with ``force_reload=True``,
|
||||||
|
:meth:`clear` or :meth:`set_rules` with
|
||||||
|
``overwrite=True`` is called this will be overwritten.
|
||||||
|
:param default_rule: Default rule to use, conf.default_rule will
|
||||||
|
be used if none is specified.
|
||||||
|
:param use_conf: Whether to load rules from cache or config file.
|
||||||
|
:param overwrite: Whether to overwrite existing rules when reload rules
|
||||||
|
from config file.
|
||||||
|
"""
|
||||||
|
global _ENFORCER
|
||||||
|
if not _ENFORCER:
|
||||||
|
# https://docs.openstack.org/oslo.policy/latest/user/usage.html
|
||||||
|
_ENFORCER = policy.Enforcer(CONF,
|
||||||
|
policy_file=policy_file,
|
||||||
|
rules=rules,
|
||||||
|
default_rule=default_rule,
|
||||||
|
use_conf=use_conf,
|
||||||
|
overwrite=overwrite)
|
||||||
|
register_rules(_ENFORCER)
|
||||||
|
return _ENFORCER
|
||||||
|
|
||||||
|
|
||||||
|
def register_rules(enforcer):
|
||||||
|
enforcer.register_defaults(policies.list_rules())
|
||||||
|
|
||||||
|
|
||||||
|
def enforce(context, rule=None, target=None,
|
||||||
|
do_raise=True, exc=None, *args, **kwargs):
|
||||||
|
|
||||||
|
"""Checks authorization of a rule against the target and credentials.
|
||||||
|
|
||||||
|
:param dict context: As much information about the user performing the
|
||||||
|
action as possible.
|
||||||
|
:param rule: The rule to evaluate.
|
||||||
|
:param dict target: As much information about the object being operated
|
||||||
|
on as possible.
|
||||||
|
:param do_raise: Whether to raise an exception or not if check
|
||||||
|
fails.
|
||||||
|
:param exc: Class of the exception to raise if the check fails.
|
||||||
|
Any remaining arguments passed to :meth:`enforce` (both
|
||||||
|
positional and keyword arguments) will be passed to
|
||||||
|
the exception class. If not specified,
|
||||||
|
:class:`PolicyNotAuthorized` will be used.
|
||||||
|
|
||||||
|
:return: ``False`` if the policy does not allow the action and `exc` is
|
||||||
|
not provided; otherwise, returns a value that evaluates to
|
||||||
|
``True``. Note: for rules using the "case" expression, this
|
||||||
|
``True`` value will be the specified string from the
|
||||||
|
expression.
|
||||||
|
"""
|
||||||
|
enforcer = init()
|
||||||
|
credentials = context.to_policy_values()
|
||||||
|
if not exc:
|
||||||
|
exc = exception.PolicyNotAuthorized
|
||||||
|
if target is None:
|
||||||
|
target = {'project_id': context.project_id,
|
||||||
|
'user_id': context.user_id}
|
||||||
|
return enforcer.enforce(rule, target, credentials,
|
||||||
|
do_raise=do_raise, exc=exc, *args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def authorize(context, action, target, do_raise=True, exc=None,
|
||||||
|
might_not_exist=False):
|
||||||
|
"""Verifies that the action is valid on the target in this context.
|
||||||
|
|
||||||
|
:param context: gyan context
|
||||||
|
:param action: string representing the action to be checked
|
||||||
|
this should be colon separated for clarity.
|
||||||
|
i.e. ``network:attach_external_network``
|
||||||
|
:param target: dictionary representing the object of the action
|
||||||
|
for object creation this should be a dictionary representing the
|
||||||
|
location of the object e.g. ``{'project_id': context.project_id}``
|
||||||
|
:param do_raise: if True (the default), raises PolicyNotAuthorized;
|
||||||
|
if False, returns False
|
||||||
|
:param exc: Class of the exception to raise if the check fails.
|
||||||
|
Any remaining arguments passed to :meth:`authorize` (both
|
||||||
|
positional and keyword arguments) will be passed to
|
||||||
|
the exception class. If not specified,
|
||||||
|
:class:`PolicyNotAuthorized` will be used.
|
||||||
|
:param might_not_exist: If True the policy check is skipped (and the
|
||||||
|
function returns True) if the specified policy does not exist.
|
||||||
|
Defaults to false.
|
||||||
|
|
||||||
|
:raises gyan.common.exception.PolicyNotAuthorized: if verification fails
|
||||||
|
and do_raise is True. Or if 'exc' is specified it will raise an
|
||||||
|
exception of that type.
|
||||||
|
|
||||||
|
:return: returns a non-False value (not necessarily "True") if
|
||||||
|
authorized, and the exact value False if not authorized and
|
||||||
|
do_raise is False.
|
||||||
|
"""
|
||||||
|
credentials = context.to_policy_values()
|
||||||
|
if not exc:
|
||||||
|
exc = exception.PolicyNotAuthorized
|
||||||
|
if might_not_exist and not (_ENFORCER.rules and action in _ENFORCER.rules):
|
||||||
|
return True
|
||||||
|
try:
|
||||||
|
result = _ENFORCER.enforce(action, target, credentials,
|
||||||
|
do_raise=do_raise, exc=exc, action=action)
|
||||||
|
except Exception:
|
||||||
|
with excutils.save_and_reraise_exception():
|
||||||
|
LOG.debug('Policy check for %(action)s failed with credentials '
|
||||||
|
'%(credentials)s',
|
||||||
|
{'action': action, 'credentials': credentials})
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def check_is_admin(context):
|
||||||
|
"""Whether or not user is admin according to policy setting.
|
||||||
|
|
||||||
|
"""
|
||||||
|
init()
|
||||||
|
target = {}
|
||||||
|
credentials = context.to_policy_values()
|
||||||
|
return _ENFORCER.enforce('context_is_admin', target, credentials)
|
||||||
22
gyan/common/privileged.py
Normal file
22
gyan/common/privileged.py
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from oslo_privsep import capabilities as c
|
||||||
|
from oslo_privsep import priv_context
|
||||||
|
|
||||||
|
|
||||||
|
default = priv_context.PrivContext(
|
||||||
|
'gyan.common',
|
||||||
|
cfg_section='privsep',
|
||||||
|
pypath=__name__ + '.default',
|
||||||
|
capabilities=[c.CAP_SYS_ADMIN],
|
||||||
|
)
|
||||||
101
gyan/common/profiler.py
Normal file
101
gyan/common/profiler.py
Normal file
@ -0,0 +1,101 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
###
|
||||||
|
# This code is taken from nova. Goal is minimal modification.
|
||||||
|
###
|
||||||
|
|
||||||
|
from oslo_log import log as logging
|
||||||
|
from oslo_utils import importutils
|
||||||
|
import webob.dec
|
||||||
|
|
||||||
|
from gyan.common import context
|
||||||
|
import gyan.conf
|
||||||
|
|
||||||
|
profiler = importutils.try_import("osprofiler.profiler")
|
||||||
|
profiler_initializer = importutils.try_import("osprofiler.initializer")
|
||||||
|
profiler_web = importutils.try_import("osprofiler.web")
|
||||||
|
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class WsgiMiddleware(object):
|
||||||
|
|
||||||
|
def __init__(self, application, **kwargs):
|
||||||
|
self.application = application
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def factory(cls, global_conf, **local_conf):
|
||||||
|
if profiler_web:
|
||||||
|
return profiler_web.WsgiMiddleware.factory(global_conf,
|
||||||
|
**local_conf)
|
||||||
|
|
||||||
|
def filter_(app):
|
||||||
|
return cls(app, **local_conf)
|
||||||
|
|
||||||
|
return filter_
|
||||||
|
|
||||||
|
@webob.dec.wsgify
|
||||||
|
def __call__(self, request):
|
||||||
|
return request.get_response(self.application)
|
||||||
|
|
||||||
|
|
||||||
|
def setup(binary, host):
|
||||||
|
if profiler_initializer and CONF.profiler.enabled:
|
||||||
|
profiler_initializer.init_from_conf(
|
||||||
|
conf=CONF,
|
||||||
|
context=context.get_admin_context().to_dict(),
|
||||||
|
project="gyan",
|
||||||
|
service=binary,
|
||||||
|
host=host)
|
||||||
|
LOG.info('OSProfiler is enabled.')
|
||||||
|
|
||||||
|
|
||||||
|
def trace_cls(name, **kwargs):
|
||||||
|
"""Wrap the OSprofiler trace_cls.
|
||||||
|
|
||||||
|
Wrap the OSprofiler trace_cls decorator so that it will not try to
|
||||||
|
patch the class unless OSprofiler is present.
|
||||||
|
|
||||||
|
:param name: The name of action. For example, wsgi, rpc, db, ...
|
||||||
|
:param kwargs: Any other keyword args used by profiler.trace_cls
|
||||||
|
"""
|
||||||
|
|
||||||
|
def decorator(cls):
|
||||||
|
if profiler and 'profiler' in CONF:
|
||||||
|
trace_decorator = profiler.trace_cls(name, kwargs)
|
||||||
|
return trace_decorator(cls)
|
||||||
|
return cls
|
||||||
|
|
||||||
|
return decorator
|
||||||
|
|
||||||
|
|
||||||
|
def trace(name, **kwargs):
|
||||||
|
"""Wrap the OSprofiler trace.
|
||||||
|
|
||||||
|
Wrap the OSprofiler trace decorator so that it will not try to
|
||||||
|
patch the functions unless OSprofiler is present.
|
||||||
|
|
||||||
|
:param name: The name of action. For example, wsgi, rpc, db, ...
|
||||||
|
:param kwargs: Any other keyword args used by profiler.trace
|
||||||
|
"""
|
||||||
|
|
||||||
|
def decorator(f):
|
||||||
|
if profiler and 'profiler' in CONF:
|
||||||
|
trace_decorator = profiler.trace(name, kwargs)
|
||||||
|
return trace_decorator(f)
|
||||||
|
return f
|
||||||
|
|
||||||
|
return decorator
|
||||||
120
gyan/common/rpc.py
Normal file
120
gyan/common/rpc.py
Normal file
@ -0,0 +1,120 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
'init',
|
||||||
|
'set_defaults',
|
||||||
|
'add_extra_exmods',
|
||||||
|
'clear_extra_exmods',
|
||||||
|
'get_allowed_exmods',
|
||||||
|
'RequestContextSerializer',
|
||||||
|
'get_client',
|
||||||
|
]
|
||||||
|
|
||||||
|
import oslo_messaging as messaging
|
||||||
|
from oslo_serialization import jsonutils as json
|
||||||
|
from oslo_utils import importutils
|
||||||
|
|
||||||
|
from gyan.common import context as gyan_context
|
||||||
|
from gyan.common import exception
|
||||||
|
|
||||||
|
profiler = importutils.try_import("osprofiler.profiler")
|
||||||
|
|
||||||
|
TRANSPORT = None
|
||||||
|
ALLOWED_EXMODS = [
|
||||||
|
exception.__name__,
|
||||||
|
]
|
||||||
|
EXTRA_EXMODS = []
|
||||||
|
|
||||||
|
|
||||||
|
def init(conf):
|
||||||
|
global TRANSPORT
|
||||||
|
exmods = get_allowed_exmods()
|
||||||
|
TRANSPORT = messaging.get_rpc_transport(
|
||||||
|
conf, allowed_remote_exmods=exmods)
|
||||||
|
|
||||||
|
|
||||||
|
def set_defaults(control_exchange):
|
||||||
|
messaging.set_transport_defaults(control_exchange)
|
||||||
|
|
||||||
|
|
||||||
|
def add_extra_exmods(*args):
|
||||||
|
EXTRA_EXMODS.extend(args)
|
||||||
|
|
||||||
|
|
||||||
|
def clear_extra_exmods():
|
||||||
|
del EXTRA_EXMODS[:]
|
||||||
|
|
||||||
|
|
||||||
|
def get_allowed_exmods():
|
||||||
|
return ALLOWED_EXMODS + EXTRA_EXMODS
|
||||||
|
|
||||||
|
|
||||||
|
class JsonPayloadSerializer(messaging.NoOpSerializer):
|
||||||
|
@staticmethod
|
||||||
|
def serialize_entity(context, entity):
|
||||||
|
return json.to_primitive(entity, convert_instances=True)
|
||||||
|
|
||||||
|
|
||||||
|
class RequestContextSerializer(messaging.Serializer):
|
||||||
|
|
||||||
|
def __init__(self, base):
|
||||||
|
self._base = base
|
||||||
|
|
||||||
|
def serialize_entity(self, context, entity):
|
||||||
|
if not self._base:
|
||||||
|
return entity
|
||||||
|
return self._base.serialize_entity(context, entity)
|
||||||
|
|
||||||
|
def deserialize_entity(self, context, entity):
|
||||||
|
if not self._base:
|
||||||
|
return entity
|
||||||
|
return self._base.deserialize_entity(context, entity)
|
||||||
|
|
||||||
|
def serialize_context(self, context):
|
||||||
|
return context.to_dict()
|
||||||
|
|
||||||
|
def deserialize_context(self, context):
|
||||||
|
return gyan_context.RequestContext.from_dict(context)
|
||||||
|
|
||||||
|
|
||||||
|
class ProfilerRequestContextSerializer(RequestContextSerializer):
|
||||||
|
def serialize_context(self, context):
|
||||||
|
_context = super(ProfilerRequestContextSerializer,
|
||||||
|
self).serialize_context(context)
|
||||||
|
|
||||||
|
prof = profiler.get()
|
||||||
|
if prof:
|
||||||
|
trace_info = {
|
||||||
|
"hmac_key": prof.hmac_key,
|
||||||
|
"base_id": prof.get_base_id(),
|
||||||
|
"parent_id": prof.get_id()
|
||||||
|
}
|
||||||
|
_context.update({"trace_info": trace_info})
|
||||||
|
|
||||||
|
return _context
|
||||||
|
|
||||||
|
def deserialize_context(self, context):
|
||||||
|
trace_info = context.pop("trace_info", None)
|
||||||
|
if trace_info:
|
||||||
|
profiler.init(**trace_info)
|
||||||
|
|
||||||
|
return super(ProfilerRequestContextSerializer,
|
||||||
|
self).deserialize_context(context)
|
||||||
|
|
||||||
|
|
||||||
|
def get_client(target, serializer=None, timeout=None):
|
||||||
|
assert TRANSPORT is not None
|
||||||
|
return messaging.RPCClient(TRANSPORT,
|
||||||
|
target,
|
||||||
|
serializer=serializer,
|
||||||
|
timeout=timeout)
|
||||||
105
gyan/common/rpc_service.py
Normal file
105
gyan/common/rpc_service.py
Normal file
@ -0,0 +1,105 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
"""Common RPC service and API tools for Gyan."""
|
||||||
|
|
||||||
|
import oslo_messaging as messaging
|
||||||
|
from oslo_messaging.rpc import dispatcher
|
||||||
|
from oslo_service import service
|
||||||
|
from oslo_utils import importutils
|
||||||
|
|
||||||
|
from gyan.common import context
|
||||||
|
from gyan.common import profiler
|
||||||
|
from gyan.common import rpc
|
||||||
|
import gyan.conf
|
||||||
|
from gyan.objects import base as objects_base
|
||||||
|
from gyan.servicegroup import gyan_service_periodic as servicegroup
|
||||||
|
|
||||||
|
osprofiler = importutils.try_import("osprofiler.profiler")
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
|
||||||
|
|
||||||
|
def _init_serializer():
|
||||||
|
serializer = rpc.RequestContextSerializer(
|
||||||
|
objects_base.GyanObjectSerializer())
|
||||||
|
if osprofiler:
|
||||||
|
serializer = rpc.ProfilerRequestContextSerializer(serializer)
|
||||||
|
else:
|
||||||
|
serializer = rpc.RequestContextSerializer(serializer)
|
||||||
|
return serializer
|
||||||
|
|
||||||
|
|
||||||
|
class Service(service.Service):
|
||||||
|
|
||||||
|
def __init__(self, topic, server, endpoints, binary):
|
||||||
|
super(Service, self).__init__()
|
||||||
|
serializer = _init_serializer()
|
||||||
|
transport = messaging.get_rpc_transport(CONF)
|
||||||
|
access_policy = dispatcher.DefaultRPCAccessPolicy
|
||||||
|
# TODO(asalkeld) add support for version='x.y'
|
||||||
|
target = messaging.Target(topic=topic, server=server)
|
||||||
|
self.endpoints = endpoints
|
||||||
|
self._server = messaging.get_rpc_server(transport, target, endpoints,
|
||||||
|
executor='eventlet',
|
||||||
|
serializer=serializer,
|
||||||
|
access_policy=access_policy)
|
||||||
|
self.binary = binary
|
||||||
|
profiler.setup(binary, CONF.compute.host)
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
servicegroup.setup(CONF, self.binary, self.tg)
|
||||||
|
for endpoint in self.endpoints:
|
||||||
|
if hasattr(endpoint, 'init_containers'):
|
||||||
|
endpoint.init_containers(
|
||||||
|
context.get_admin_context(all_projects=True))
|
||||||
|
self.tg.add_dynamic_timer(
|
||||||
|
endpoint.run_periodic_tasks,
|
||||||
|
periodic_interval_max=CONF.periodic_interval_max,
|
||||||
|
context=context.get_admin_context(all_projects=True)
|
||||||
|
)
|
||||||
|
self._server.start()
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
if self._server:
|
||||||
|
self._server.stop()
|
||||||
|
self._server.wait()
|
||||||
|
super(Service, self).stop()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def create(cls, topic, server, handlers, binary):
|
||||||
|
service_obj = cls(topic, server, handlers, binary)
|
||||||
|
return service_obj
|
||||||
|
|
||||||
|
|
||||||
|
class API(object):
|
||||||
|
def __init__(self, context=None, topic=None, server=None,
|
||||||
|
timeout=None):
|
||||||
|
serializer = _init_serializer()
|
||||||
|
self._context = context
|
||||||
|
if topic is None:
|
||||||
|
topic = ''
|
||||||
|
target = messaging.Target(topic=topic, server=server)
|
||||||
|
self._client = rpc.get_client(target,
|
||||||
|
serializer=serializer,
|
||||||
|
timeout=timeout)
|
||||||
|
|
||||||
|
def _call(self, server, method, *args, **kwargs):
|
||||||
|
cctxt = self._client.prepare(server=server)
|
||||||
|
return cctxt.call(self._context, method, *args, **kwargs)
|
||||||
|
|
||||||
|
def _cast(self, server, method, *args, **kwargs):
|
||||||
|
cctxt = self._client.prepare(server=server)
|
||||||
|
return cctxt.cast(self._context, method, *args, **kwargs)
|
||||||
|
|
||||||
|
def echo(self, message):
|
||||||
|
self._cast('echo', message=message)
|
||||||
92
gyan/common/service.py
Normal file
92
gyan/common/service.py
Normal file
@ -0,0 +1,92 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
|
||||||
|
from oslo_concurrency import processutils
|
||||||
|
from oslo_log import log
|
||||||
|
from oslo_service import service
|
||||||
|
from oslo_service import wsgi
|
||||||
|
|
||||||
|
from gyan.api import app
|
||||||
|
from gyan.common import config
|
||||||
|
from gyan.common import exception
|
||||||
|
from gyan.common.i18n import _
|
||||||
|
import gyan.conf
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_service(argv=None):
|
||||||
|
if argv is None:
|
||||||
|
argv = []
|
||||||
|
log.register_options(CONF)
|
||||||
|
config.parse_args(argv)
|
||||||
|
config.set_config_defaults()
|
||||||
|
log.setup(CONF, 'gyan')
|
||||||
|
# TODO(yuanying): Uncomment after objects are implemented
|
||||||
|
# objects.register_all()
|
||||||
|
|
||||||
|
|
||||||
|
def process_launcher():
|
||||||
|
return service.ProcessLauncher(CONF, restart_method='mutate')
|
||||||
|
|
||||||
|
|
||||||
|
class WSGIService(service.ServiceBase):
|
||||||
|
"""Provides ability to launch Gyan API from wsgi app."""
|
||||||
|
|
||||||
|
def __init__(self, name, use_ssl=False):
|
||||||
|
"""Initialize, but do not start the WSGI server.
|
||||||
|
|
||||||
|
:param name: The name of the WSGI server given to the loader.
|
||||||
|
:param use_ssl: Wraps the socket in an SSL context if True.
|
||||||
|
:returns: None
|
||||||
|
"""
|
||||||
|
self.name = name
|
||||||
|
self.app = app.load_app()
|
||||||
|
self.workers = (CONF.api.workers or processutils.get_worker_count())
|
||||||
|
if self.workers and self.workers < 1:
|
||||||
|
raise exception.ConfigInvalid(
|
||||||
|
_("api_workers value of %d is invalid, "
|
||||||
|
"must be greater than 0.") % self.workers)
|
||||||
|
|
||||||
|
self.server = wsgi.Server(CONF, name, self.app,
|
||||||
|
host=CONF.api.host_ip,
|
||||||
|
port=CONF.api.port,
|
||||||
|
use_ssl=use_ssl)
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
"""Start serving this service using loaded configuration.
|
||||||
|
|
||||||
|
:returns: None
|
||||||
|
"""
|
||||||
|
self.server.start()
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
"""Stop serving this API.
|
||||||
|
|
||||||
|
:returns: None
|
||||||
|
"""
|
||||||
|
self.server.stop()
|
||||||
|
|
||||||
|
def wait(self):
|
||||||
|
"""Wait for the service to stop serving this API.
|
||||||
|
|
||||||
|
:returns: None
|
||||||
|
"""
|
||||||
|
self.server.wait()
|
||||||
|
|
||||||
|
def reset(self):
|
||||||
|
"""Reset server greenpool size to default.
|
||||||
|
|
||||||
|
:returns: None
|
||||||
|
"""
|
||||||
|
self.server.reset()
|
||||||
63
gyan/common/short_id.py
Normal file
63
gyan/common/short_id.py
Normal file
@ -0,0 +1,63 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""Utilities for creating short ID strings based on a random UUID.
|
||||||
|
The IDs each comprise 12 (lower-case) alphanumeric characters.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import base64
|
||||||
|
from oslo_utils import uuidutils
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
import six
|
||||||
|
|
||||||
|
from gyan.common.i18n import _
|
||||||
|
|
||||||
|
|
||||||
|
def _to_byte_string(value, num_bits):
|
||||||
|
"""Convert an integer to a big-endian string of bytes with padding.
|
||||||
|
|
||||||
|
Padding is added at the end (i.e. after the least-significant bit) if
|
||||||
|
required.
|
||||||
|
"""
|
||||||
|
|
||||||
|
shifts = six.moves.xrange(num_bits - 8, -8, -8)
|
||||||
|
byte_at = lambda off: (value >> off if off >= 0 else value << -off) & 0xff
|
||||||
|
return ''.join(six.int2byte(byte_at(offset)) for offset in shifts)
|
||||||
|
|
||||||
|
|
||||||
|
def get_id(source_uuid):
|
||||||
|
"""Derive a short (12 character) id from a random UUID.
|
||||||
|
|
||||||
|
The supplied UUID must be a version 4 UUID object.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if isinstance(source_uuid, six.string_types):
|
||||||
|
source_uuid = uuid.UUID(source_uuid)
|
||||||
|
if source_uuid.version != 4:
|
||||||
|
raise ValueError(_('Invalid UUID version (%d)') % source_uuid.version)
|
||||||
|
|
||||||
|
# The "time" field of a v4 UUID contains 60 random bits
|
||||||
|
# (see RFC4122, Section 4.4)
|
||||||
|
random_bytes = _to_byte_string(source_uuid.time, 60)
|
||||||
|
# The first 12 bytes (= 60 bits) of base32-encoded output is our data
|
||||||
|
encoded = base64.b32encode(six.b(random_bytes))[:12]
|
||||||
|
|
||||||
|
if six.PY3:
|
||||||
|
return encoded.lower().decode('utf-8')
|
||||||
|
else:
|
||||||
|
return encoded.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def generate_id():
|
||||||
|
"""Generate a short (12 character), random id."""
|
||||||
|
return uuidutils.generate_uuid()
|
||||||
25
gyan/common/singleton.py
Normal file
25
gyan/common/singleton.py
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from oslo_concurrency import lockutils
|
||||||
|
|
||||||
|
|
||||||
|
class Singleton(type):
|
||||||
|
_instances = {}
|
||||||
|
_semaphores = lockutils.Semaphores()
|
||||||
|
|
||||||
|
def __call__(cls, *args, **kwargs):
|
||||||
|
with lockutils.lock('singleton_lock', semaphores=cls._semaphores):
|
||||||
|
if cls not in cls._instances:
|
||||||
|
cls._instances[cls] = super(
|
||||||
|
Singleton, cls).__call__(*args, **kwargs)
|
||||||
|
return cls._instances[cls]
|
||||||
255
gyan/common/utils.py
Normal file
255
gyan/common/utils.py
Normal file
@ -0,0 +1,255 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
# It's based on oslo.i18n usage in OpenStack Keystone project and
|
||||||
|
# recommendations from
|
||||||
|
# https://docs.openstack.org/oslo.i18n/latest/user/usage.html
|
||||||
|
|
||||||
|
"""Utilities and helper functions."""
|
||||||
|
import base64
|
||||||
|
import binascii
|
||||||
|
import eventlet
|
||||||
|
import functools
|
||||||
|
import inspect
|
||||||
|
import json
|
||||||
|
import mimetypes
|
||||||
|
|
||||||
|
from oslo_concurrency import processutils
|
||||||
|
from oslo_context import context as common_context
|
||||||
|
from oslo_log import log as logging
|
||||||
|
from oslo_utils import excutils
|
||||||
|
from oslo_utils import strutils
|
||||||
|
import pecan
|
||||||
|
import six
|
||||||
|
|
||||||
|
from gyan.api import utils as api_utils
|
||||||
|
from gyan.common import consts
|
||||||
|
from gyan.common import exception
|
||||||
|
from gyan.common.i18n import _
|
||||||
|
from gyan.common import privileged
|
||||||
|
import gyan.conf
|
||||||
|
from gyan import objects
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
VALID_STATES = {
|
||||||
|
'deploy': [consts.CREATED, consts.UNDEPLOYED],
|
||||||
|
'undeploy': [consts.DEPLOYED]
|
||||||
|
}
|
||||||
|
def safe_rstrip(value, chars=None):
|
||||||
|
"""Removes trailing characters from a string if that does not make it empty
|
||||||
|
|
||||||
|
:param value: A string value that will be stripped.
|
||||||
|
:param chars: Characters to remove.
|
||||||
|
:return: Stripped value.
|
||||||
|
|
||||||
|
"""
|
||||||
|
if not isinstance(value, six.string_types):
|
||||||
|
LOG.warning(
|
||||||
|
"Failed to remove trailing character. Returning original object. "
|
||||||
|
"Supplied object is not a string: %s.", value)
|
||||||
|
return value
|
||||||
|
|
||||||
|
return value.rstrip(chars) or value
|
||||||
|
|
||||||
|
|
||||||
|
def _do_allow_certain_content_types(func, content_types_list):
|
||||||
|
# Allows you to bypass pecan's content-type restrictions
|
||||||
|
cfg = pecan.util._cfg(func)
|
||||||
|
cfg.setdefault('content_types', {})
|
||||||
|
cfg['content_types'].update((value, '')
|
||||||
|
for value in content_types_list)
|
||||||
|
return func
|
||||||
|
|
||||||
|
|
||||||
|
def allow_certain_content_types(*content_types_list):
|
||||||
|
def _wrapper(func):
|
||||||
|
return _do_allow_certain_content_types(func, content_types_list)
|
||||||
|
return _wrapper
|
||||||
|
|
||||||
|
|
||||||
|
def allow_all_content_types(f):
|
||||||
|
return _do_allow_certain_content_types(f, mimetypes.types_map.values())
|
||||||
|
|
||||||
|
|
||||||
|
def spawn_n(func, *args, **kwargs):
|
||||||
|
"""Passthrough method for eventlet.spawn_n.
|
||||||
|
|
||||||
|
This utility exists so that it can be stubbed for testing without
|
||||||
|
interfering with the service spawns.
|
||||||
|
|
||||||
|
It will also grab the context from the threadlocal store and add it to
|
||||||
|
the store on the new thread. This allows for continuity in logging the
|
||||||
|
context when using this method to spawn a new thread.
|
||||||
|
"""
|
||||||
|
_context = common_context.get_current()
|
||||||
|
|
||||||
|
@functools.wraps(func)
|
||||||
|
def context_wrapper(*args, **kwargs):
|
||||||
|
# NOTE: If update_store is not called after spawn_n it won't be
|
||||||
|
# available for the logger to pull from threadlocal storage.
|
||||||
|
if _context is not None:
|
||||||
|
_context.update_store()
|
||||||
|
func(*args, **kwargs)
|
||||||
|
|
||||||
|
eventlet.spawn_n(context_wrapper, *args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def translate_exception(function):
|
||||||
|
"""Wraps a method to catch exceptions.
|
||||||
|
|
||||||
|
If the exception is not an instance of GyanException,
|
||||||
|
translate it into one.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@functools.wraps(function)
|
||||||
|
def decorated_function(self, context, *args, **kwargs):
|
||||||
|
try:
|
||||||
|
return function(self, context, *args, **kwargs)
|
||||||
|
except Exception as e:
|
||||||
|
if not isinstance(e, exception.GyanException):
|
||||||
|
LOG.exception("Unexpected error: %s", six.text_type(e))
|
||||||
|
e = exception.GyanException("Unexpected error: %s"
|
||||||
|
% six.text_type(e))
|
||||||
|
raise e
|
||||||
|
raise
|
||||||
|
|
||||||
|
return decorated_function
|
||||||
|
|
||||||
|
|
||||||
|
def custom_execute(*cmd, **kwargs):
|
||||||
|
try:
|
||||||
|
return processutils.execute(*cmd, **kwargs)
|
||||||
|
except processutils.ProcessExecutionError as e:
|
||||||
|
sanitized_cmd = strutils.mask_password(' '.join(cmd))
|
||||||
|
raise exception.CommandError(cmd=sanitized_cmd,
|
||||||
|
error=six.text_type(e))
|
||||||
|
|
||||||
|
|
||||||
|
def is_all_projects(search_opts):
|
||||||
|
all_projects = search_opts.get('all_projects')
|
||||||
|
if all_projects:
|
||||||
|
try:
|
||||||
|
all_projects = strutils.bool_from_string(all_projects, True)
|
||||||
|
except ValueError:
|
||||||
|
bools = ', '.join(strutils.TRUE_STRINGS + strutils.FALSE_STRINGS)
|
||||||
|
raise exception.InvalidValue(_('Valid all_projects values are: %s')
|
||||||
|
% bools)
|
||||||
|
else:
|
||||||
|
all_projects = False
|
||||||
|
return all_projects
|
||||||
|
|
||||||
|
|
||||||
|
def get_ml_model(ml_model_ident):
|
||||||
|
ml_model = api_utils.get_resource('ML_Model', ml_model_ident)
|
||||||
|
if not ml_model:
|
||||||
|
pecan.abort(404, ('Not found; the ml model you requested '
|
||||||
|
'does not exist.'))
|
||||||
|
|
||||||
|
return ml_model
|
||||||
|
|
||||||
|
def validate_ml_model_state(ml_model, action):
|
||||||
|
if ml_model.status not in VALID_STATES[action]:
|
||||||
|
raise exception.InvalidStateException(
|
||||||
|
id=ml_model.uuid,
|
||||||
|
action=action,
|
||||||
|
actual_state=ml_model.status)
|
||||||
|
|
||||||
|
|
||||||
|
def get_wrapped_function(function):
|
||||||
|
"""Get the method at the bottom of a stack of decorators."""
|
||||||
|
if not hasattr(function, '__closure__') or not function.__closure__:
|
||||||
|
return function
|
||||||
|
|
||||||
|
def _get_wrapped_function(function):
|
||||||
|
if not hasattr(function, '__closure__') or not function.__closure__:
|
||||||
|
return None
|
||||||
|
|
||||||
|
for closure in function.__closure__:
|
||||||
|
func = closure.cell_contents
|
||||||
|
|
||||||
|
deeper_func = _get_wrapped_function(func)
|
||||||
|
if deeper_func:
|
||||||
|
return deeper_func
|
||||||
|
elif hasattr(closure.cell_contents, '__call__'):
|
||||||
|
return closure.cell_contents
|
||||||
|
|
||||||
|
return function
|
||||||
|
|
||||||
|
return _get_wrapped_function(function)
|
||||||
|
|
||||||
|
|
||||||
|
def wrap_ml_model_event(prefix):
|
||||||
|
"""Warps a method to log the event taken on the ml_model, and result.
|
||||||
|
|
||||||
|
This decorator wraps a method to log the start and result of an event, as
|
||||||
|
part of an action taken on a ml_model.
|
||||||
|
"""
|
||||||
|
def helper(function):
|
||||||
|
|
||||||
|
@functools.wraps(function)
|
||||||
|
def decorated_function(self, context, *args, **kwargs):
|
||||||
|
wrapped_func = get_wrapped_function(function)
|
||||||
|
keyed_args = inspect.getcallargs(wrapped_func, self, context,
|
||||||
|
*args, **kwargs)
|
||||||
|
ml_model_uuid = keyed_args['ml_model'].uuid
|
||||||
|
|
||||||
|
event_name = '{0}_{1}'.format(prefix, function.__name__)
|
||||||
|
with EventReporter(context, event_name, ml_model_uuid):
|
||||||
|
return function(self, context, *args, **kwargs)
|
||||||
|
return decorated_function
|
||||||
|
return helper
|
||||||
|
|
||||||
|
|
||||||
|
def wrap_exception():
|
||||||
|
def helper(function):
|
||||||
|
|
||||||
|
@functools.wraps(function)
|
||||||
|
def decorated_function(self, context, ml_model, *args, **kwargs):
|
||||||
|
try:
|
||||||
|
return function(self, context, ml_model, *args, **kwargs)
|
||||||
|
except exception.DockerError as e:
|
||||||
|
with excutils.save_and_reraise_exception(reraise=False):
|
||||||
|
LOG.error("Error occurred while calling Docker API: %s",
|
||||||
|
six.text_type(e))
|
||||||
|
except Exception as e:
|
||||||
|
with excutils.save_and_reraise_exception(reraise=False):
|
||||||
|
LOG.exception("Unexpected exception: %s", six.text_type(e))
|
||||||
|
return decorated_function
|
||||||
|
return helper
|
||||||
|
|
||||||
|
|
||||||
|
def is_close(x, y, rel_tol=1e-06, abs_tol=0.0):
|
||||||
|
return abs(x - y) <= max(rel_tol * max(abs(x), abs(y)), abs_tol)
|
||||||
|
|
||||||
|
|
||||||
|
def is_less_than(x, y):
|
||||||
|
if isinstance(x, int) and isinstance(y, int):
|
||||||
|
return x < y
|
||||||
|
if isinstance(x, float) or isinstance(y, float):
|
||||||
|
return False if (x - y) >= 0 or is_close(x, y) else True
|
||||||
|
|
||||||
|
|
||||||
|
def encode_file_data(data):
|
||||||
|
if six.PY3 and isinstance(data, str):
|
||||||
|
data = data.encode('utf-8')
|
||||||
|
return base64.b64encode(data).decode('utf-8')
|
||||||
|
|
||||||
|
|
||||||
|
def decode_file_data(data):
|
||||||
|
# Py3 raises binascii.Error instead of TypeError as in Py27
|
||||||
|
try:
|
||||||
|
return base64.b64decode(data)
|
||||||
|
except (TypeError, binascii.Error):
|
||||||
|
raise exception.Base64Exception()
|
||||||
33
gyan/common/yamlutils.py
Normal file
33
gyan/common/yamlutils.py
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
|
||||||
|
def load(s):
|
||||||
|
try:
|
||||||
|
yml_dict = yaml.safe_load(s)
|
||||||
|
except yaml.YAMLError as exc:
|
||||||
|
msg = 'An error occurred during YAML parsing.'
|
||||||
|
if hasattr(exc, 'problem_mark'):
|
||||||
|
msg += ' Error position: (%s:%s)' % (exc.problem_mark.line + 1,
|
||||||
|
exc.problem_mark.column + 1)
|
||||||
|
raise ValueError(msg)
|
||||||
|
if not isinstance(yml_dict, dict) and not isinstance(yml_dict, list):
|
||||||
|
raise ValueError('The source is not a YAML mapping or list.')
|
||||||
|
if isinstance(yml_dict, dict) and len(yml_dict) < 1:
|
||||||
|
raise ValueError('Could not find any element in your YAML mapping.')
|
||||||
|
return yml_dict
|
||||||
|
|
||||||
|
|
||||||
|
def dump(s):
|
||||||
|
return yaml.safe_dump(s)
|
||||||
0
gyan/compute/__init__.py
Normal file
0
gyan/compute/__init__.py
Normal file
63
gyan/compute/api.py
Normal file
63
gyan/compute/api.py
Normal file
@ -0,0 +1,63 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""Handles all requests relating to compute resources (e.g. ml_models,
|
||||||
|
and compute hosts on which they run)."""
|
||||||
|
|
||||||
|
from oslo_log import log as logging
|
||||||
|
|
||||||
|
from gyan.common import consts
|
||||||
|
from gyan.common import exception
|
||||||
|
from gyan.common.i18n import _
|
||||||
|
from gyan.common import profiler
|
||||||
|
from gyan.compute import rpcapi
|
||||||
|
import gyan.conf
|
||||||
|
from gyan import objects
|
||||||
|
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@profiler.trace_cls("rpc")
|
||||||
|
class API(object):
|
||||||
|
"""API for interacting with the compute manager."""
|
||||||
|
|
||||||
|
def __init__(self, context):
|
||||||
|
self.rpcapi = rpcapi.API(context=context)
|
||||||
|
super(API, self).__init__()
|
||||||
|
|
||||||
|
def ml_model_create(self, context, new_ml_model, extra_spec):
|
||||||
|
try:
|
||||||
|
host_state = self._schedule_ml_model(context, ml_model,
|
||||||
|
extra_spec)
|
||||||
|
except exception.NoValidHost:
|
||||||
|
new_ml_model.status = consts.ERROR
|
||||||
|
new_ml_model.status_reason = _(
|
||||||
|
"There are not enough hosts available.")
|
||||||
|
new_ml_model.save(context)
|
||||||
|
return
|
||||||
|
except Exception:
|
||||||
|
new_ml_model.status = consts.ERROR
|
||||||
|
new_ml_model.status_reason = _("Unexpected exception occurred.")
|
||||||
|
new_ml_model.save(context)
|
||||||
|
raise
|
||||||
|
|
||||||
|
self.rpcapi.ml_model_create(context, host_state['host'],
|
||||||
|
new_ml_model)
|
||||||
|
|
||||||
|
def ml_model_delete(self, context, ml_model, *args):
|
||||||
|
self._record_action_start(context, ml_model, ml_model_actions.DELETE)
|
||||||
|
return self.rpcapi.ml_model_delete(context, ml_model, *args)
|
||||||
|
|
||||||
|
def ml_model_show(self, context, ml_model):
|
||||||
|
return self.rpcapi.ml_model_show(context, ml_model)
|
||||||
67
gyan/compute/compute_host_tracker.py
Normal file
67
gyan/compute/compute_host_tracker.py
Normal file
@ -0,0 +1,67 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import collections
|
||||||
|
import copy
|
||||||
|
|
||||||
|
from oslo_log import log as logging
|
||||||
|
|
||||||
|
from gyan.common import exception
|
||||||
|
from gyan.common import utils
|
||||||
|
from gyan import objects
|
||||||
|
from gyan.objects import base as obj_base
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
COMPUTE_RESOURCE_SEMAPHORE = "compute_resources"
|
||||||
|
|
||||||
|
|
||||||
|
class ComputeHostTracker(object):
|
||||||
|
def __init__(self, host, ml_model_driver):
|
||||||
|
self.host = host
|
||||||
|
self.ml_model_driver = ml_model_driver
|
||||||
|
self.compute_host = None
|
||||||
|
self.tracked_ml_models = {}
|
||||||
|
self.old_resources = collections.defaultdict(objects.ComputeHost)
|
||||||
|
|
||||||
|
|
||||||
|
def update_available_resources(self, context):
|
||||||
|
# Check if the compute_host is already registered
|
||||||
|
host = self._get_compute_host(context)
|
||||||
|
if not host:
|
||||||
|
# If not, register it and pass the object to the driver
|
||||||
|
host = objects.ComputeHost(context)
|
||||||
|
host.hostname = self.host
|
||||||
|
host.type = self.ml_model_driver.__class__.__name__
|
||||||
|
host.status = "AVAILABLE"
|
||||||
|
host.create(context)
|
||||||
|
LOG.info('Host created for :%(host)s', {'host': self.host})
|
||||||
|
self.ml_model_driver.get_available_resources(host)
|
||||||
|
self.compute_host = host
|
||||||
|
return host
|
||||||
|
|
||||||
|
def _get_compute_host(self, context):
|
||||||
|
"""Returns compute host for the host"""
|
||||||
|
try:
|
||||||
|
return objects.ComputeHost.get_by_name(context, self.host)
|
||||||
|
except exception.ComputeHostNotFound:
|
||||||
|
LOG.warning("No compute host record for: %(host)s",
|
||||||
|
{'host': self.host})
|
||||||
|
|
||||||
|
|
||||||
|
def _set_ml_model_host(self, context, ml_model):
|
||||||
|
"""Tag the ml_model as belonging to this host.
|
||||||
|
|
||||||
|
This should be done while the COMPUTE_RESOURCES_SEMAPHORE is held so
|
||||||
|
the resource claim will not be lost if the audit process starts.
|
||||||
|
"""
|
||||||
|
ml_model.host = self.host
|
||||||
|
ml_model.save(context)
|
||||||
121
gyan/compute/manager.py
Normal file
121
gyan/compute/manager.py
Normal file
@ -0,0 +1,121 @@
|
|||||||
|
# Copyright 2016 IBM Corp.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import itertools
|
||||||
|
|
||||||
|
import six
|
||||||
|
import time
|
||||||
|
|
||||||
|
from oslo_log import log as logging
|
||||||
|
from oslo_service import periodic_task
|
||||||
|
from oslo_utils import excutils
|
||||||
|
from oslo_utils import timeutils
|
||||||
|
from oslo_utils import uuidutils
|
||||||
|
|
||||||
|
from gyan.common import consts
|
||||||
|
from gyan.common import context
|
||||||
|
from gyan.common import exception
|
||||||
|
from gyan.common.i18n import _
|
||||||
|
from gyan.common import utils
|
||||||
|
from gyan.common.utils import translate_exception
|
||||||
|
from gyan.common.utils import wrap_ml_model_event
|
||||||
|
from gyan.common.utils import wrap_exception
|
||||||
|
from gyan.compute import compute_host_tracker
|
||||||
|
import gyan.conf
|
||||||
|
from gyan.ml_model import driver
|
||||||
|
from gyan import objects
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class Manager(periodic_task.PeriodicTasks):
|
||||||
|
"""Manages the running ml_models."""
|
||||||
|
|
||||||
|
def __init__(self, ml_model_driver=None):
|
||||||
|
super(Manager, self).__init__(CONF)
|
||||||
|
self.driver = driver.load_ml_model_driver(ml_model_driver)
|
||||||
|
self.host = CONF.compute.host
|
||||||
|
self._resource_tracker = None
|
||||||
|
|
||||||
|
def ml_model_create(self, context, limits, requested_networks,
|
||||||
|
requested_volumes, ml_model, run, pci_requests=None):
|
||||||
|
@utils.synchronized(ml_model.uuid)
|
||||||
|
def do_ml_model_create():
|
||||||
|
created_ml_model = self._do_ml_model_create(
|
||||||
|
context, ml_model, requested_networks, requested_volumes,
|
||||||
|
pci_requests, limits)
|
||||||
|
if run:
|
||||||
|
self._do_ml_model_start(context, created_ml_model)
|
||||||
|
|
||||||
|
utils.spawn_n(do_ml_model_create)
|
||||||
|
|
||||||
|
@wrap_ml_model_event(prefix='compute')
|
||||||
|
def _do_ml_model_create(self, context, ml_model, requested_networks,
|
||||||
|
requested_volumes, pci_requests=None,
|
||||||
|
limits=None):
|
||||||
|
LOG.debug('Creating ml_model: %s', ml_model.uuid)
|
||||||
|
|
||||||
|
try:
|
||||||
|
rt = self._get_resource_tracker()
|
||||||
|
# As sriov port also need to claim, we need claim pci port before
|
||||||
|
# create sandbox.
|
||||||
|
with rt.ml_model_claim(context, ml_model, pci_requests, limits):
|
||||||
|
sandbox = None
|
||||||
|
if self.use_sandbox:
|
||||||
|
sandbox = self._create_sandbox(context, ml_model,
|
||||||
|
requested_networks)
|
||||||
|
|
||||||
|
created_ml_model = self._do_ml_model_create_base(
|
||||||
|
context, ml_model, requested_networks, requested_volumes,
|
||||||
|
sandbox, limits)
|
||||||
|
return created_ml_model
|
||||||
|
except exception.ResourcesUnavailable as e:
|
||||||
|
with excutils.save_and_reraise_exception():
|
||||||
|
LOG.exception("ML Model resource claim failed: %s",
|
||||||
|
six.text_type(e))
|
||||||
|
self._fail_ml_model(context, ml_model, six.text_type(e),
|
||||||
|
unset_host=True)
|
||||||
|
|
||||||
|
@wrap_ml_model_event(prefix='compute')
|
||||||
|
def _do_ml_model_start(self, context, ml_model):
|
||||||
|
pass
|
||||||
|
|
||||||
|
@translate_exception
|
||||||
|
def ml_model_delete(self, context, ml_model, force=False):
|
||||||
|
pass
|
||||||
|
|
||||||
|
@translate_exception
|
||||||
|
def ml_model_show(self, context, ml_model):
|
||||||
|
pass
|
||||||
|
|
||||||
|
@translate_exception
|
||||||
|
def ml_model_start(self, context, ml_model):
|
||||||
|
pass
|
||||||
|
|
||||||
|
@translate_exception
|
||||||
|
def ml_model_update(self, context, ml_model, patch):
|
||||||
|
pass
|
||||||
|
|
||||||
|
@periodic_task.periodic_task(run_immediately=True)
|
||||||
|
def inventory_host(self, context):
|
||||||
|
rt = self._get_resource_tracker()
|
||||||
|
rt.update_available_resources(context)
|
||||||
|
|
||||||
|
def _get_resource_tracker(self):
|
||||||
|
if not self._resource_tracker:
|
||||||
|
rt = compute_host_tracker.ComputeHostTracker(self.host,
|
||||||
|
self.driver)
|
||||||
|
self._resource_tracker = rt
|
||||||
|
return self._resource_tracker
|
||||||
68
gyan/compute/rpcapi.py
Normal file
68
gyan/compute/rpcapi.py
Normal file
@ -0,0 +1,68 @@
|
|||||||
|
# Copyright 2016 IBM Corp.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import functools
|
||||||
|
|
||||||
|
from gyan.api import servicegroup
|
||||||
|
from gyan.common import exception
|
||||||
|
from gyan.common import profiler
|
||||||
|
from gyan.common import rpc_service
|
||||||
|
import gyan.conf
|
||||||
|
from gyan import objects
|
||||||
|
|
||||||
|
|
||||||
|
def check_ml_model_host(func):
|
||||||
|
"""Verify the state of ML Model host"""
|
||||||
|
@functools.wraps(func)
|
||||||
|
def wrap(self, context, ml_model, *args, **kwargs):
|
||||||
|
return func(self, context, ml_model, *args, **kwargs)
|
||||||
|
return wrap
|
||||||
|
|
||||||
|
|
||||||
|
@profiler.trace_cls("rpc")
|
||||||
|
class API(rpc_service.API):
|
||||||
|
"""Client side of the ml_model compute rpc API.
|
||||||
|
|
||||||
|
API version history:
|
||||||
|
|
||||||
|
* 1.0 - Initial version.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, transport=None, context=None, topic=None):
|
||||||
|
if topic is None:
|
||||||
|
gyan.conf.CONF.import_opt(
|
||||||
|
'topic', 'gyan.conf.compute', group='compute')
|
||||||
|
|
||||||
|
super(API, self).__init__(
|
||||||
|
context, gyan.conf.CONF.compute.topic, transport)
|
||||||
|
|
||||||
|
def ml_model_create(self, context, host, ml_model):
|
||||||
|
self._cast(host, 'ml_model_create',
|
||||||
|
ml_model=ml_model)
|
||||||
|
|
||||||
|
@check_ml_model_host
|
||||||
|
def ml_model_delete(self, context, ml_model, force):
|
||||||
|
return self._cast(ml_model.host, 'ml_model_delete',
|
||||||
|
ml_model=ml_model, force=force)
|
||||||
|
|
||||||
|
@check_ml_model_host
|
||||||
|
def ml_model_show(self, context, ml_model):
|
||||||
|
return self._call(ml_model.host, 'ml_model_show',
|
||||||
|
ml_model=ml_model)
|
||||||
|
|
||||||
|
|
||||||
|
@check_ml_model_host
|
||||||
|
def ml_model_update(self, context, ml_model, patch):
|
||||||
|
return self._call(ml_model.host, 'ml_model_update',
|
||||||
|
ml_model=ml_model, patch=patch)
|
||||||
41
gyan/conf/__init__.py
Normal file
41
gyan/conf/__init__.py
Normal file
@ -0,0 +1,41 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
|
||||||
|
from gyan.conf import api
|
||||||
|
from gyan.conf import compute
|
||||||
|
from gyan.conf import ml_model_driver
|
||||||
|
from gyan.conf import database
|
||||||
|
from gyan.conf import keystone
|
||||||
|
from gyan.conf import path
|
||||||
|
from gyan.conf import profiler
|
||||||
|
from gyan.conf import scheduler
|
||||||
|
from gyan.conf import services
|
||||||
|
from gyan.conf import ssl
|
||||||
|
from gyan.conf import utils
|
||||||
|
from gyan.conf import gyan_client
|
||||||
|
|
||||||
|
CONF = cfg.CONF
|
||||||
|
|
||||||
|
api.register_opts(CONF)
|
||||||
|
compute.register_opts(CONF)
|
||||||
|
ml_model_driver.register_opts(CONF)
|
||||||
|
database.register_opts(CONF)
|
||||||
|
keystone.register_opts(CONF)
|
||||||
|
path.register_opts(CONF)
|
||||||
|
scheduler.register_opts(CONF)
|
||||||
|
services.register_opts(CONF)
|
||||||
|
gyan_client.register_opts(CONF)
|
||||||
|
ssl.register_opts(CONF)
|
||||||
|
profiler.register_opts(CONF)
|
||||||
|
utils.register_opts(CONF)
|
||||||
67
gyan/conf/api.py
Normal file
67
gyan/conf/api.py
Normal file
@ -0,0 +1,67 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
|
||||||
|
|
||||||
|
api_service_opts = [
|
||||||
|
cfg.PortOpt('port',
|
||||||
|
default=8517,
|
||||||
|
help='The port for the gyan API server.'),
|
||||||
|
cfg.StrOpt('host',
|
||||||
|
default="localhost",
|
||||||
|
help='The port for the gyan API server.'),
|
||||||
|
cfg.IPOpt('host_ip',
|
||||||
|
default='127.0.0.1',
|
||||||
|
help="The listen IP for the gyan API server. "
|
||||||
|
"The default is ``$my_ip``, "
|
||||||
|
"the IP address of this host."),
|
||||||
|
cfg.BoolOpt('enable_ssl_api',
|
||||||
|
default=False,
|
||||||
|
help="Enable the integrated stand-alone API to service "
|
||||||
|
"requests via HTTPS instead of HTTP. If there is a "
|
||||||
|
"front-end service performing HTTPS offloading from "
|
||||||
|
"the service, this option should be False; note, you "
|
||||||
|
"will want to change public API endpoint to represent "
|
||||||
|
"SSL termination URL with 'public_endpoint' option."),
|
||||||
|
cfg.IntOpt('workers',
|
||||||
|
help="Number of workers for gyan-api service. "
|
||||||
|
"The default will be the number of CPUs available."),
|
||||||
|
cfg.IntOpt('max_limit',
|
||||||
|
default=1000,
|
||||||
|
help='The maximum number of items returned in a single '
|
||||||
|
'response from a collection resource.'),
|
||||||
|
cfg.StrOpt('api_paste_config',
|
||||||
|
default="api-paste.ini",
|
||||||
|
help="Configuration file for WSGI definition of API."),
|
||||||
|
cfg.BoolOpt('enable_image_validation',
|
||||||
|
default=True,
|
||||||
|
help="Enable image validation.")
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
api_group = cfg.OptGroup(name='api',
|
||||||
|
title='Options for the gyan-api service')
|
||||||
|
|
||||||
|
|
||||||
|
ALL_OPTS = (api_service_opts)
|
||||||
|
|
||||||
|
|
||||||
|
def register_opts(conf):
|
||||||
|
conf.register_group(api_group)
|
||||||
|
conf.register_opts(ALL_OPTS, api_group)
|
||||||
|
|
||||||
|
|
||||||
|
def list_opts():
|
||||||
|
return {
|
||||||
|
api_group: ALL_OPTS
|
||||||
|
}
|
||||||
40
gyan/conf/compute.py
Normal file
40
gyan/conf/compute.py
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
|
||||||
|
|
||||||
|
service_opts = [
|
||||||
|
cfg.StrOpt(
|
||||||
|
'topic',
|
||||||
|
default='gyan-compute',
|
||||||
|
help='The queue to add compute tasks to.'),
|
||||||
|
cfg.StrOpt(
|
||||||
|
'host',
|
||||||
|
default='localhost',
|
||||||
|
help='hostname'),
|
||||||
|
|
||||||
|
]
|
||||||
|
|
||||||
|
opt_group = cfg.OptGroup(
|
||||||
|
name='compute', title='Options for the gyan-compute service')
|
||||||
|
|
||||||
|
ALL_OPTS = (service_opts)
|
||||||
|
|
||||||
|
|
||||||
|
def register_opts(conf):
|
||||||
|
conf.register_group(opt_group)
|
||||||
|
conf.register_opts(ALL_OPTS, opt_group)
|
||||||
|
|
||||||
|
|
||||||
|
def list_opts():
|
||||||
|
return {opt_group: ALL_OPTS}
|
||||||
32
gyan/conf/database.py
Normal file
32
gyan/conf/database.py
Normal file
@ -0,0 +1,32 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
|
||||||
|
|
||||||
|
sql_opts = [
|
||||||
|
cfg.StrOpt('mysql_engine',
|
||||||
|
default='InnoDB',
|
||||||
|
help='MySQL engine to use.')
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_OPTS = (sql_opts)
|
||||||
|
|
||||||
|
|
||||||
|
def register_opts(conf):
|
||||||
|
conf.register_opts(sql_opts, 'database')
|
||||||
|
|
||||||
|
|
||||||
|
def list_opts():
|
||||||
|
return {"DEFAULT": DEFAULT_OPTS}
|
||||||
51
gyan/conf/gyan_client.py
Normal file
51
gyan/conf/gyan_client.py
Normal file
@ -0,0 +1,51 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
|
||||||
|
|
||||||
|
gyan_group = cfg.OptGroup(name='gyan_client',
|
||||||
|
title='Options for the Gyan client')
|
||||||
|
|
||||||
|
gyan_client_opts = [
|
||||||
|
cfg.StrOpt('region_name',
|
||||||
|
help='Region in Identity service catalog to use for '
|
||||||
|
'communication with the OpenStack service.'),
|
||||||
|
cfg.StrOpt('endpoint_type',
|
||||||
|
default='publicURL',
|
||||||
|
help='Type of endpoint in Identity service catalog to use '
|
||||||
|
'for communication with the OpenStack service.')]
|
||||||
|
|
||||||
|
|
||||||
|
common_security_opts = [
|
||||||
|
cfg.StrOpt('ca_file',
|
||||||
|
help='Optional CA cert file to use in SSL connections.'),
|
||||||
|
cfg.StrOpt('cert_file',
|
||||||
|
help='Optional PEM-formatted certificate chain file.'),
|
||||||
|
cfg.StrOpt('key_file',
|
||||||
|
help='Optional PEM-formatted file that contains the '
|
||||||
|
'private key.'),
|
||||||
|
cfg.BoolOpt('insecure',
|
||||||
|
default=False,
|
||||||
|
help="If set, then the server's certificate will not "
|
||||||
|
"be verified.")]
|
||||||
|
|
||||||
|
ALL_OPTS = (gyan_client_opts + common_security_opts)
|
||||||
|
|
||||||
|
|
||||||
|
def register_opts(conf):
|
||||||
|
conf.register_group(gyan_group)
|
||||||
|
conf.register_opts(gyan_client_opts, group=gyan_group)
|
||||||
|
|
||||||
|
|
||||||
|
def list_opts():
|
||||||
|
return {gyan_group: ALL_OPTS}
|
||||||
35
gyan/conf/keystone.py
Normal file
35
gyan/conf/keystone.py
Normal file
@ -0,0 +1,35 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
||||||
|
# use this file except in compliance with the License. You may obtain a copy
|
||||||
|
# of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from keystoneauth1 import loading as ka_loading
|
||||||
|
from oslo_config import cfg
|
||||||
|
|
||||||
|
CFG_GROUP = 'keystone_auth'
|
||||||
|
CFG_LEGACY_GROUP = 'keystone_authtoken'
|
||||||
|
|
||||||
|
keystone_auth_group = cfg.OptGroup(name=CFG_GROUP,
|
||||||
|
title='Options for Keystone in Gyan')
|
||||||
|
|
||||||
|
|
||||||
|
def register_opts(conf):
|
||||||
|
conf.import_group(CFG_LEGACY_GROUP, 'keystonemiddleware.auth_token')
|
||||||
|
ka_loading.register_auth_conf_options(conf, CFG_GROUP)
|
||||||
|
ka_loading.register_session_conf_options(conf, CFG_GROUP)
|
||||||
|
conf.set_default('auth_type', default='password', group=CFG_GROUP)
|
||||||
|
|
||||||
|
|
||||||
|
def list_opts():
|
||||||
|
keystone_auth_opts = (ka_loading.get_auth_common_conf_options() +
|
||||||
|
ka_loading.get_auth_plugin_conf_options('password'))
|
||||||
|
return {
|
||||||
|
keystone_auth_group: keystone_auth_opts
|
||||||
|
}
|
||||||
47
gyan/conf/ml_model_driver.py
Normal file
47
gyan/conf/ml_model_driver.py
Normal file
@ -0,0 +1,47 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
|
||||||
|
driver_opts = [
|
||||||
|
cfg.StrOpt('ml_model_driver',
|
||||||
|
default='gyan.ml_model.tensorflow.driver.TensorflowDriver',
|
||||||
|
help="""Defines which driver to use for controlling ml_model.
|
||||||
|
Possible values:
|
||||||
|
|
||||||
|
* ``ml_model.driver.TensorflowDriver``
|
||||||
|
|
||||||
|
Services which consume this:
|
||||||
|
|
||||||
|
* ``gyan-compute``
|
||||||
|
|
||||||
|
Interdependencies to other options:
|
||||||
|
|
||||||
|
* None
|
||||||
|
"""),
|
||||||
|
cfg.IntOpt('default_sleep_time', default=1,
|
||||||
|
help='Time to sleep (in seconds) during waiting for an event.'),
|
||||||
|
cfg.IntOpt('default_timeout', default=60 * 10,
|
||||||
|
help='Maximum time (in seconds) to wait for an event.')
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
ALL_OPTS = (driver_opts)
|
||||||
|
|
||||||
|
|
||||||
|
def register_opts(conf):
|
||||||
|
conf.register_opts(ALL_OPTS)
|
||||||
|
|
||||||
|
|
||||||
|
def list_opts():
|
||||||
|
return {"DEFAULT": ALL_OPTS}
|
||||||
76
gyan/conf/opts.py
Normal file
76
gyan/conf/opts.py
Normal file
@ -0,0 +1,76 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""
|
||||||
|
This is the single point of entry to generate the sample configuration
|
||||||
|
file for Gyan. It collects all the necessary info from the other modules
|
||||||
|
in this package. It is assumed that:
|
||||||
|
|
||||||
|
* every other module in this package has a 'list_opts' function which
|
||||||
|
return a dict where
|
||||||
|
* the keys are strings which are the group names
|
||||||
|
* the value of each key is a list of config options for that group
|
||||||
|
* the gyan.conf package doesn't have further packages with config options
|
||||||
|
* this module is only used in the context of sample file generation
|
||||||
|
"""
|
||||||
|
|
||||||
|
import collections
|
||||||
|
import importlib
|
||||||
|
import os
|
||||||
|
import pkgutil
|
||||||
|
|
||||||
|
LIST_OPTS_FUNC_NAME = "list_opts"
|
||||||
|
|
||||||
|
|
||||||
|
def _tupleize(dct):
|
||||||
|
"""Take the dict of options and convert to the 2-tuple format."""
|
||||||
|
return [(key, val) for key, val in dct.items()]
|
||||||
|
|
||||||
|
|
||||||
|
def list_opts():
|
||||||
|
opts = collections.defaultdict(list)
|
||||||
|
module_names = _list_module_names()
|
||||||
|
imported_modules = _import_modules(module_names)
|
||||||
|
_append_config_options(imported_modules, opts)
|
||||||
|
return _tupleize(opts)
|
||||||
|
|
||||||
|
|
||||||
|
def _list_module_names():
|
||||||
|
module_names = []
|
||||||
|
package_path = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
for _, modname, ispkg in pkgutil.iter_modules(path=[package_path]):
|
||||||
|
if modname == "opts" or ispkg:
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
module_names.append(modname)
|
||||||
|
return module_names
|
||||||
|
|
||||||
|
|
||||||
|
def _import_modules(module_names):
|
||||||
|
imported_modules = []
|
||||||
|
for modname in module_names:
|
||||||
|
mod = importlib.import_module("gyan.conf." + modname)
|
||||||
|
if not hasattr(mod, LIST_OPTS_FUNC_NAME):
|
||||||
|
msg = "The module 'gyan.conf.%s' should have a '%s' "\
|
||||||
|
"function which returns the config options." % \
|
||||||
|
(modname, LIST_OPTS_FUNC_NAME)
|
||||||
|
raise AttributeError(msg)
|
||||||
|
else:
|
||||||
|
imported_modules.append(mod)
|
||||||
|
return imported_modules
|
||||||
|
|
||||||
|
|
||||||
|
def _append_config_options(imported_modules, config_options):
|
||||||
|
for mod in imported_modules:
|
||||||
|
configs = mod.list_opts()
|
||||||
|
for key, val in configs.items():
|
||||||
|
config_options[key].extend(val)
|
||||||
42
gyan/conf/path.py
Normal file
42
gyan/conf/path.py
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
|
||||||
|
|
||||||
|
path_opts = [
|
||||||
|
cfg.StrOpt('pybasedir',
|
||||||
|
default=os.path.abspath(os.path.join(os.path.dirname(__file__),
|
||||||
|
'../')),
|
||||||
|
help='Directory where the gyan python module is installed.'),
|
||||||
|
cfg.StrOpt('bindir',
|
||||||
|
default='$pybasedir/bin',
|
||||||
|
help='Directory where gyan binaries are installed.'),
|
||||||
|
cfg.StrOpt('state_path',
|
||||||
|
default='$pybasedir',
|
||||||
|
help="Top-level directory for maintaining gyan's state."),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def state_path_def(*args):
|
||||||
|
"""Return an uninterpolated path relative to $state_path."""
|
||||||
|
return os.path.join('$state_path', *args)
|
||||||
|
|
||||||
|
|
||||||
|
def register_opts(conf):
|
||||||
|
conf.register_opts(path_opts)
|
||||||
|
|
||||||
|
|
||||||
|
def list_opts():
|
||||||
|
return {"DEFAULT": path_opts}
|
||||||
29
gyan/conf/profiler.py
Normal file
29
gyan/conf/profiler.py
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
||||||
|
# use this file except in compliance with the License. You may obtain a copy
|
||||||
|
# of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from oslo_utils import importutils
|
||||||
|
|
||||||
|
|
||||||
|
profiler_opts = importutils.try_import('osprofiler.opts')
|
||||||
|
|
||||||
|
|
||||||
|
def register_opts(conf):
|
||||||
|
if profiler_opts:
|
||||||
|
profiler_opts.set_defaults(conf)
|
||||||
|
|
||||||
|
|
||||||
|
def list_opts():
|
||||||
|
if not profiler_opts:
|
||||||
|
return {}
|
||||||
|
return {
|
||||||
|
profiler_opts._profiler_opt_group: profiler_opts._PROFILER_OPTS
|
||||||
|
}
|
||||||
104
gyan/conf/scheduler.py
Normal file
104
gyan/conf/scheduler.py
Normal file
@ -0,0 +1,104 @@
|
|||||||
|
# Copyright 2015 OpenStack Foundation
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
|
||||||
|
|
||||||
|
scheduler_group = cfg.OptGroup(name="scheduler",
|
||||||
|
title="Scheduler configuration")
|
||||||
|
|
||||||
|
scheduler_opts = [
|
||||||
|
cfg.StrOpt("driver",
|
||||||
|
default="filter_scheduler",
|
||||||
|
choices=("chance_scheduler", "fake_scheduler",
|
||||||
|
"filter_scheduler"),
|
||||||
|
help="""
|
||||||
|
The class of the driver used by the scheduler.
|
||||||
|
|
||||||
|
The options are chosen from the entry points under the namespace
|
||||||
|
'gyan.scheduler.driver' in 'setup.cfg'.
|
||||||
|
|
||||||
|
Possible values:
|
||||||
|
|
||||||
|
* A string, where the string corresponds to the class name of a scheduler
|
||||||
|
driver. There are a number of options available:
|
||||||
|
** 'chance_scheduler', which simply picks a host at random
|
||||||
|
** A custom scheduler driver. In this case, you will be responsible for
|
||||||
|
creating and maintaining the entry point in your 'setup.cfg' file
|
||||||
|
"""),
|
||||||
|
cfg.MultiStrOpt("available_filters",
|
||||||
|
default=["gyan.scheduler.filters.all_filters"],
|
||||||
|
help="""
|
||||||
|
Filters that the scheduler can use.
|
||||||
|
|
||||||
|
An unordered list of the filter classes the gyan scheduler may apply. Only the
|
||||||
|
filters specified in the 'scheduler_enabled_filters' option will be used, but
|
||||||
|
any filter appearing in that option must also be included in this list.
|
||||||
|
|
||||||
|
By default, this is set to all filters that are included with gyan.
|
||||||
|
|
||||||
|
This option is only used by the FilterScheduler and its subclasses; if you use
|
||||||
|
a different scheduler, this option has no effect.
|
||||||
|
|
||||||
|
Possible values:
|
||||||
|
|
||||||
|
* A list of zero or more strings, where each string corresponds to the name of
|
||||||
|
a filter that may be used for selecting a host
|
||||||
|
|
||||||
|
Related options:
|
||||||
|
|
||||||
|
* scheduler_enabled_filters
|
||||||
|
"""),
|
||||||
|
cfg.ListOpt("enabled_filters",
|
||||||
|
default=[
|
||||||
|
"AvailabilityZoneFilter",
|
||||||
|
"CPUFilter",
|
||||||
|
"RamFilter",
|
||||||
|
"ComputeFilter",
|
||||||
|
"DiskFilter",
|
||||||
|
],
|
||||||
|
help="""
|
||||||
|
Filters that the scheduler will use.
|
||||||
|
|
||||||
|
An ordered list of filter class names that will be used for filtering
|
||||||
|
hosts. Ignore the word 'default' in the name of this option: these filters will
|
||||||
|
*always* be applied, and they will be applied in the order they are listed so
|
||||||
|
place your most restrictive filters first to make the filtering process more
|
||||||
|
efficient.
|
||||||
|
|
||||||
|
This option is only used by the FilterScheduler and its subclasses; if you use
|
||||||
|
a different scheduler, this option has no effect.
|
||||||
|
|
||||||
|
Possible values:
|
||||||
|
|
||||||
|
* A list of zero or more strings, where each string corresponds to the name of
|
||||||
|
a filter to be used for selecting a host
|
||||||
|
|
||||||
|
Related options:
|
||||||
|
|
||||||
|
* All of the filters in this option *must* be present in the
|
||||||
|
'scheduler_available_filters' option, or a SchedulerHostFilterNotFound
|
||||||
|
exception will be raised.
|
||||||
|
"""),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def register_opts(conf):
|
||||||
|
conf.register_group(scheduler_group)
|
||||||
|
conf.register_opts(scheduler_opts, group=scheduler_group)
|
||||||
|
|
||||||
|
|
||||||
|
def list_opts():
|
||||||
|
return {scheduler_group: scheduler_opts}
|
||||||
36
gyan/conf/services.py
Normal file
36
gyan/conf/services.py
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
|
||||||
|
|
||||||
|
periodic_opts = [
|
||||||
|
cfg.IntOpt('periodic_interval_max',
|
||||||
|
default=60,
|
||||||
|
help='Max interval size between periodic tasks execution in '
|
||||||
|
'seconds.'),
|
||||||
|
cfg.IntOpt('service_down_time',
|
||||||
|
default=180,
|
||||||
|
help='Max interval size between periodic tasks execution in '
|
||||||
|
'seconds.')
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
ALL_OPTS = (periodic_opts)
|
||||||
|
|
||||||
|
|
||||||
|
def register_opts(conf):
|
||||||
|
conf.register_opts(ALL_OPTS)
|
||||||
|
|
||||||
|
|
||||||
|
def list_opts():
|
||||||
|
return {"DEFAULT": ALL_OPTS}
|
||||||
27
gyan/conf/ssl.py
Normal file
27
gyan/conf/ssl.py
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
from oslo_service import sslutils
|
||||||
|
|
||||||
|
|
||||||
|
def register_opts(conf):
|
||||||
|
sslutils.register_opts(conf)
|
||||||
|
|
||||||
|
|
||||||
|
def list_opts():
|
||||||
|
group_name, ssl_opts = sslutils.list_opts()[0]
|
||||||
|
ssl_group = cfg.OptGroup(name=group_name,
|
||||||
|
title='Options for the ssl')
|
||||||
|
return {
|
||||||
|
ssl_group: ssl_opts
|
||||||
|
}
|
||||||
31
gyan/conf/utils.py
Normal file
31
gyan/conf/utils.py
Normal file
@ -0,0 +1,31 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
||||||
|
# use this file except in compliance with the License. You may obtain a copy
|
||||||
|
# of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
|
||||||
|
|
||||||
|
utils_opts = [
|
||||||
|
cfg.StrOpt('rootwrap_config',
|
||||||
|
default="/etc/gyan/rootwrap.conf",
|
||||||
|
help='Path to the rootwrap configuration file to use for '
|
||||||
|
'running commands as root.'),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def register_opts(conf):
|
||||||
|
conf.register_opts(utils_opts)
|
||||||
|
|
||||||
|
|
||||||
|
def list_opts():
|
||||||
|
return {
|
||||||
|
"DEFAULT": utils_opts
|
||||||
|
}
|
||||||
21
gyan/db/__init__.py
Normal file
21
gyan/db/__init__.py
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from oslo_db import options
|
||||||
|
|
||||||
|
from gyan.common import paths
|
||||||
|
import gyan.conf
|
||||||
|
|
||||||
|
_DEFAULT_SQL_CONNECTION = 'sqlite:///' + paths.state_path_def('gyan.sqlite')
|
||||||
|
|
||||||
|
options.set_defaults(gyan.conf.CONF)
|
||||||
|
options.set_defaults(gyan.conf.CONF, _DEFAULT_SQL_CONNECTION)
|
||||||
197
gyan/db/api.py
Normal file
197
gyan/db/api.py
Normal file
@ -0,0 +1,197 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
"""
|
||||||
|
Base API for Database
|
||||||
|
"""
|
||||||
|
|
||||||
|
from oslo_db import api as db_api
|
||||||
|
|
||||||
|
from gyan.common import profiler
|
||||||
|
import gyan.conf
|
||||||
|
|
||||||
|
"""Add the database backend mapping here"""
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
_BACKEND_MAPPING = {'sqlalchemy': 'gyan.db.sqlalchemy.api'}
|
||||||
|
IMPL = db_api.DBAPI.from_config(CONF,
|
||||||
|
backend_mapping=_BACKEND_MAPPING,
|
||||||
|
lazy=True)
|
||||||
|
|
||||||
|
|
||||||
|
@profiler.trace("db")
|
||||||
|
def _get_dbdriver_instance():
|
||||||
|
"""Return a DB API instance."""
|
||||||
|
return IMPL
|
||||||
|
|
||||||
|
|
||||||
|
@profiler.trace("db")
|
||||||
|
def list_ml_models(context, filters=None, limit=None, marker=None,
|
||||||
|
sort_key=None, sort_dir=None):
|
||||||
|
"""List matching ML Models.
|
||||||
|
|
||||||
|
Return a list of the specified columns for all ml models that match
|
||||||
|
the specified filters.
|
||||||
|
|
||||||
|
:param context: The security context
|
||||||
|
:param filters: Filters to apply. Defaults to None.
|
||||||
|
:param limit: Maximum number of ml_models to return.
|
||||||
|
:param marker: the last item of the previous page; we return the next
|
||||||
|
result set.
|
||||||
|
:param sort_key: Attribute by which results should be sorted.
|
||||||
|
:param sort_dir: Direction in which results should be sorted.
|
||||||
|
(asc, desc)
|
||||||
|
:returns: A list of tuples of the specified columns.
|
||||||
|
"""
|
||||||
|
return _get_dbdriver_instance().list_ml_models(
|
||||||
|
context, filters, limit, marker, sort_key, sort_dir)
|
||||||
|
|
||||||
|
|
||||||
|
@profiler.trace("db")
|
||||||
|
def create_ml_model(context, values):
|
||||||
|
"""Create a new ML Model.
|
||||||
|
|
||||||
|
:param context: The security context
|
||||||
|
:param values: A dict containing several items used to identify
|
||||||
|
and track the ML Model
|
||||||
|
:returns: A ML Model.
|
||||||
|
"""
|
||||||
|
return _get_dbdriver_instance().create_ml_model(context, values)
|
||||||
|
|
||||||
|
|
||||||
|
@profiler.trace("db")
|
||||||
|
def get_ml_model_by_uuid(context, ml_model_uuid):
|
||||||
|
"""Return a ML Model.
|
||||||
|
|
||||||
|
:param context: The security context
|
||||||
|
:param ml_model_uuid: The uuid of a ml model.
|
||||||
|
:returns: A ML Model.
|
||||||
|
"""
|
||||||
|
return _get_dbdriver_instance().get_ml_model_by_uuid(
|
||||||
|
context, ml_model_uuid)
|
||||||
|
|
||||||
|
|
||||||
|
@profiler.trace("db")
|
||||||
|
def get_ml_model_by_name(context, ml_model_name):
|
||||||
|
"""Return a ML Model.
|
||||||
|
|
||||||
|
:param context: The security context
|
||||||
|
:param ml_model_name: The name of a ML Model.
|
||||||
|
:returns: A ML Model.
|
||||||
|
"""
|
||||||
|
return _get_dbdriver_instance().get_ml_model_by_name(
|
||||||
|
context, ml_model_name)
|
||||||
|
|
||||||
|
|
||||||
|
@profiler.trace("db")
|
||||||
|
def destroy_ml_model(context, ml_model_id):
|
||||||
|
"""Destroy a ml model and all associated interfaces.
|
||||||
|
|
||||||
|
:param context: Request context
|
||||||
|
:param ml_model_id: The id or uuid of a ml model.
|
||||||
|
"""
|
||||||
|
return _get_dbdriver_instance().destroy_ml_model(context, ml_model_id)
|
||||||
|
|
||||||
|
|
||||||
|
@profiler.trace("db")
|
||||||
|
def update_ml_model(context, ml_model_id, values):
|
||||||
|
"""Update properties of a ml model.
|
||||||
|
|
||||||
|
:param context: Request context
|
||||||
|
:param ml_model_id: The id or uuid of a ml model.
|
||||||
|
:param values: The properties to be updated
|
||||||
|
:returns: A ML Model.
|
||||||
|
:raises: MLModelNotFound
|
||||||
|
"""
|
||||||
|
return _get_dbdriver_instance().update_ml_model(
|
||||||
|
context, ml_model_id, values)
|
||||||
|
|
||||||
|
|
||||||
|
@profiler.trace("db")
|
||||||
|
def list_compute_hosts(context, filters=None, limit=None, marker=None,
|
||||||
|
sort_key=None, sort_dir=None):
|
||||||
|
"""List matching compute hosts.
|
||||||
|
|
||||||
|
Return a list of the specified columns for all compute hosts that match
|
||||||
|
the specified filters.
|
||||||
|
|
||||||
|
:param context: The security context
|
||||||
|
:param filters: Filters to apply. Defaults to None.
|
||||||
|
:param limit: Maximum number of compute nodes to return.
|
||||||
|
:param marker: the last item of the previous page; we return the next
|
||||||
|
result set.
|
||||||
|
:param sort_key: Attribute by which results should be sorted.
|
||||||
|
:param sort_dir: Direction in which results should be sorted.
|
||||||
|
(asc, desc)
|
||||||
|
:returns: A list of tuples of the specified columns.
|
||||||
|
"""
|
||||||
|
return _get_dbdriver_instance().list_compute_hosts(
|
||||||
|
context, filters, limit, marker, sort_key, sort_dir)
|
||||||
|
|
||||||
|
|
||||||
|
@profiler.trace("db")
|
||||||
|
def create_compute_host(context, values):
|
||||||
|
"""Create a new compute host.
|
||||||
|
|
||||||
|
:param context: The security context
|
||||||
|
:param values: A dict containing several items used to identify
|
||||||
|
and track the compute node, and several dicts which are
|
||||||
|
passed into the Drivers when managing this compute host.
|
||||||
|
:returns: A compute host.
|
||||||
|
"""
|
||||||
|
return _get_dbdriver_instance().create_compute_host(context, values)
|
||||||
|
|
||||||
|
|
||||||
|
@profiler.trace("db")
|
||||||
|
def get_compute_host(context, host_uuid):
|
||||||
|
"""Return a compute host.
|
||||||
|
|
||||||
|
:param context: The security context
|
||||||
|
:param node_uuid: The uuid of a compute node.
|
||||||
|
:returns: A compute node.
|
||||||
|
"""
|
||||||
|
return _get_dbdriver_instance().get_compute_host(context, host_uuid)
|
||||||
|
|
||||||
|
|
||||||
|
@profiler.trace("db")
|
||||||
|
def get_compute_host_by_hostname(context, hostname):
|
||||||
|
"""Return a compute node.
|
||||||
|
|
||||||
|
:param context: The security context
|
||||||
|
:param hostname: The hostname of a compute node.
|
||||||
|
:returns: A compute node.
|
||||||
|
"""
|
||||||
|
return _get_dbdriver_instance().get_compute_host_by_hostname(
|
||||||
|
context, hostname)
|
||||||
|
|
||||||
|
|
||||||
|
@profiler.trace("db")
|
||||||
|
def destroy_compute_host(context, host_uuid):
|
||||||
|
"""Destroy a compute node and all associated interfaces.
|
||||||
|
|
||||||
|
:param context: Request context
|
||||||
|
:param node_uuid: The uuid of a compute node.
|
||||||
|
"""
|
||||||
|
return _get_dbdriver_instance().destroy_compute_host(context, host_uuid)
|
||||||
|
|
||||||
|
|
||||||
|
@profiler.trace("db")
|
||||||
|
def update_compute_host(context, host_uuid, values):
|
||||||
|
"""Update properties of a compute node.
|
||||||
|
|
||||||
|
:param context: Request context
|
||||||
|
:param node_uuid: The uuid of a compute node.
|
||||||
|
:param values: The properties to be updated
|
||||||
|
:returns: A compute node.
|
||||||
|
:raises: ComputeNodeNotFound
|
||||||
|
"""
|
||||||
|
return _get_dbdriver_instance().update_compute_host(
|
||||||
|
context, host_uuid, values)
|
||||||
47
gyan/db/migration.py
Normal file
47
gyan/db/migration.py
Normal file
@ -0,0 +1,47 @@
|
|||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""Database setup and migration commands."""
|
||||||
|
|
||||||
|
from stevedore import driver
|
||||||
|
|
||||||
|
import gyan.conf
|
||||||
|
|
||||||
|
_IMPL = None
|
||||||
|
|
||||||
|
|
||||||
|
def get_backend():
|
||||||
|
global _IMPL
|
||||||
|
if not _IMPL:
|
||||||
|
gyan.conf.CONF.import_opt('backend',
|
||||||
|
'oslo_db.options', group='database')
|
||||||
|
_IMPL = driver.DriverManager("gyan.database.migration_backend",
|
||||||
|
gyan.conf.CONF.database.backend).driver
|
||||||
|
return _IMPL
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade(version=None):
|
||||||
|
"""Migrate the database to `version` or the most recent version."""
|
||||||
|
return get_backend().upgrade(version)
|
||||||
|
|
||||||
|
|
||||||
|
def version():
|
||||||
|
return get_backend().version()
|
||||||
|
|
||||||
|
|
||||||
|
def stamp(version):
|
||||||
|
return get_backend().stamp(version)
|
||||||
|
|
||||||
|
|
||||||
|
def revision(message, autogenerate):
|
||||||
|
return get_backend().revision(message, autogenerate)
|
||||||
0
gyan/db/sqlalchemy/__init__.py
Normal file
0
gyan/db/sqlalchemy/__init__.py
Normal file
68
gyan/db/sqlalchemy/alembic.ini
Normal file
68
gyan/db/sqlalchemy/alembic.ini
Normal file
@ -0,0 +1,68 @@
|
|||||||
|
# A generic, single database configuration.
|
||||||
|
|
||||||
|
[alembic]
|
||||||
|
# path to migration scripts
|
||||||
|
script_location = %(here)s/alembic
|
||||||
|
|
||||||
|
# template used to generate migration files
|
||||||
|
# file_template = %%(rev)s_%%(slug)s
|
||||||
|
|
||||||
|
# max length of characters to apply to the
|
||||||
|
# "slug" field
|
||||||
|
#truncate_slug_length = 40
|
||||||
|
|
||||||
|
# set to 'true' to run the environment during
|
||||||
|
# the 'revision' command, regardless of autogenerate
|
||||||
|
# revision_environment = false
|
||||||
|
|
||||||
|
# set to 'true' to allow .pyc and .pyo files without
|
||||||
|
# a source .py file to be detected as revisions in the
|
||||||
|
# versions/ directory
|
||||||
|
# sourceless = false
|
||||||
|
|
||||||
|
# version location specification; this defaults
|
||||||
|
# to alembic/versions. When using multiple version
|
||||||
|
# directories, initial revisions must be specified with --version-path
|
||||||
|
# version_locations = %(here)s/bar %(here)s/bat alembic/versions
|
||||||
|
|
||||||
|
# the output encoding used when revision files
|
||||||
|
# are written from script.py.mako
|
||||||
|
# output_encoding = utf-8
|
||||||
|
|
||||||
|
#sqlalchemy.url = driver://user:pass@localhost/dbname
|
||||||
|
|
||||||
|
|
||||||
|
# Logging configuration
|
||||||
|
[loggers]
|
||||||
|
keys = root,sqlalchemy,alembic
|
||||||
|
|
||||||
|
[handlers]
|
||||||
|
keys = console
|
||||||
|
|
||||||
|
[formatters]
|
||||||
|
keys = generic
|
||||||
|
|
||||||
|
[logger_root]
|
||||||
|
level = WARN
|
||||||
|
handlers = console
|
||||||
|
qualname =
|
||||||
|
|
||||||
|
[logger_sqlalchemy]
|
||||||
|
level = WARN
|
||||||
|
handlers =
|
||||||
|
qualname = sqlalchemy.engine
|
||||||
|
|
||||||
|
[logger_alembic]
|
||||||
|
level = INFO
|
||||||
|
handlers =
|
||||||
|
qualname = alembic
|
||||||
|
|
||||||
|
[handler_console]
|
||||||
|
class = StreamHandler
|
||||||
|
args = (sys.stderr,)
|
||||||
|
level = NOTSET
|
||||||
|
formatter = generic
|
||||||
|
|
||||||
|
[formatter_generic]
|
||||||
|
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||||
|
datefmt = %H:%M:%S
|
||||||
11
gyan/db/sqlalchemy/alembic/README
Normal file
11
gyan/db/sqlalchemy/alembic/README
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
Please see https://alembic.readthedocs.org/en/latest/index.html for general documentation
|
||||||
|
|
||||||
|
To create alembic migrations use:
|
||||||
|
$ gyan-db-manage revision --message "description of revision" --autogenerate
|
||||||
|
|
||||||
|
Stamp db with most recent migration version, without actually running migrations
|
||||||
|
$ gyan-db-manage stamp head
|
||||||
|
|
||||||
|
Upgrade can be performed by:
|
||||||
|
$ gyan-db-manage upgrade
|
||||||
|
$ gyan-db-manage upgrade head
|
||||||
58
gyan/db/sqlalchemy/alembic/env.py
Normal file
58
gyan/db/sqlalchemy/alembic/env.py
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
|
||||||
|
from __future__ import with_statement
|
||||||
|
from alembic import context
|
||||||
|
from logging.config import fileConfig
|
||||||
|
|
||||||
|
from gyan.db.sqlalchemy import api as sqla_api
|
||||||
|
from gyan.db.sqlalchemy import models
|
||||||
|
# this is the Alembic Config object, which provides
|
||||||
|
# access to the values within the .ini file in use.
|
||||||
|
config = context.config
|
||||||
|
|
||||||
|
# Interpret the config file for Python logging.
|
||||||
|
# This line sets up loggers basically.
|
||||||
|
fileConfig(config.config_file_name)
|
||||||
|
|
||||||
|
# add your model's MetaData object here
|
||||||
|
# for 'autogenerate' support
|
||||||
|
# from myapp import mymodel
|
||||||
|
# target_metadata = mymodel.Base.metadata
|
||||||
|
target_metadata = models.Base.metadata
|
||||||
|
|
||||||
|
# other values from the config, defined by the needs of env.py,
|
||||||
|
# can be acquired:
|
||||||
|
# my_important_option = config.get_main_option("my_important_option")
|
||||||
|
# ... etc.
|
||||||
|
|
||||||
|
|
||||||
|
def run_migrations_online():
|
||||||
|
"""Run migrations in 'online' mode.
|
||||||
|
|
||||||
|
In this scenario we need to create an Engine
|
||||||
|
and associate a connection with the context.
|
||||||
|
|
||||||
|
"""
|
||||||
|
engine = sqla_api.get_engine()
|
||||||
|
with engine.connect() as connection:
|
||||||
|
context.configure(
|
||||||
|
connection=connection,
|
||||||
|
target_metadata=target_metadata,
|
||||||
|
render_as_batch=True
|
||||||
|
)
|
||||||
|
|
||||||
|
with context.begin_transaction():
|
||||||
|
context.run_migrations()
|
||||||
|
|
||||||
|
run_migrations_online()
|
||||||
20
gyan/db/sqlalchemy/alembic/script.py.mako
Normal file
20
gyan/db/sqlalchemy/alembic/script.py.mako
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
"""${message}
|
||||||
|
|
||||||
|
Revision ID: ${up_revision}
|
||||||
|
Revises: ${down_revision | comma,n}
|
||||||
|
Create Date: ${create_date}
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = ${repr(up_revision)}
|
||||||
|
down_revision = ${repr(down_revision)}
|
||||||
|
branch_labels = ${repr(branch_labels)}
|
||||||
|
depends_on = ${repr(depends_on)}
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
${imports if imports else ""}
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
${upgrades if upgrades else "pass"}
|
||||||
@ -0,0 +1,48 @@
|
|||||||
|
"""Add Model and Host tables
|
||||||
|
|
||||||
|
Revision ID: cebd81b206ca
|
||||||
|
Create Date: 2018-10-09 09:57:20.823110
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = 'cebd81b206ca'
|
||||||
|
down_revision = None
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
op.create_table(
|
||||||
|
'compute_host',
|
||||||
|
sa.Column('id', sa.String(length=255), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||||
|
sa.Column('updated_at', sa.DateTime(), nullable=True),
|
||||||
|
sa.Column('hostname', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('status', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('type', sa.String(length=255), nullable=True),
|
||||||
|
sa.PrimaryKeyConstraint('id')
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
'ml_model',
|
||||||
|
sa.Column('id', sa.String(length=255), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||||
|
sa.Column('updated_at', sa.DateTime(), nullable=True),
|
||||||
|
sa.Column('host_id', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('name', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('project_id', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('user_id', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('status', sa.String(length=255), nullable=True),
|
||||||
|
sa.Column('status_reason', sa.Text, nullable=True),
|
||||||
|
sa.Column('url', sa.Text, nullable=True),
|
||||||
|
sa.Column('hints', sa.Text, nullable=True),
|
||||||
|
sa.Column('deployed', sa.Boolean(), nullable=True),
|
||||||
|
sa.PrimaryKeyConstraint('id'),
|
||||||
|
sa.ForeignKeyConstraint(['host_id'], ['compute_host.id'])
|
||||||
|
)
|
||||||
|
|
||||||
|
# ### end Alembic commands ###
|
||||||
306
gyan/db/sqlalchemy/api.py
Normal file
306
gyan/db/sqlalchemy/api.py
Normal file
@ -0,0 +1,306 @@
|
|||||||
|
# Copyright 2013 Hewlett-Packard Development Company, L.P.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""SQLAlchemy storage backend."""
|
||||||
|
|
||||||
|
from oslo_db import exception as db_exc
|
||||||
|
from oslo_db.sqlalchemy import session as db_session
|
||||||
|
from oslo_db.sqlalchemy import utils as db_utils
|
||||||
|
from oslo_utils import importutils
|
||||||
|
from oslo_utils import strutils
|
||||||
|
from oslo_utils import timeutils
|
||||||
|
from oslo_utils import uuidutils
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from sqlalchemy.orm import contains_eager
|
||||||
|
from sqlalchemy.orm.exc import MultipleResultsFound
|
||||||
|
from sqlalchemy.orm.exc import NoResultFound
|
||||||
|
from sqlalchemy.sql.expression import desc
|
||||||
|
from sqlalchemy.sql import func
|
||||||
|
|
||||||
|
from gyan.common import consts
|
||||||
|
from gyan.common import exception
|
||||||
|
from gyan.common.i18n import _
|
||||||
|
import gyan.conf
|
||||||
|
from gyan.db.sqlalchemy import models
|
||||||
|
|
||||||
|
profiler_sqlalchemy = importutils.try_import('osprofiler.sqlalchemy')
|
||||||
|
|
||||||
|
CONF = gyan.conf.CONF
|
||||||
|
|
||||||
|
_FACADE = None
|
||||||
|
|
||||||
|
|
||||||
|
def _create_facade_lazily():
|
||||||
|
global _FACADE
|
||||||
|
if _FACADE is None:
|
||||||
|
_FACADE = db_session.enginefacade.get_legacy_facade()
|
||||||
|
if profiler_sqlalchemy:
|
||||||
|
if CONF.profiler.enabled and CONF.profiler.trace_sqlalchemy:
|
||||||
|
profiler_sqlalchemy.add_tracing(sa, _FACADE.get_engine(), "db")
|
||||||
|
return _FACADE
|
||||||
|
|
||||||
|
|
||||||
|
def get_engine():
|
||||||
|
facade = _create_facade_lazily()
|
||||||
|
return facade.get_engine()
|
||||||
|
|
||||||
|
|
||||||
|
def get_session(**kwargs):
|
||||||
|
facade = _create_facade_lazily()
|
||||||
|
return facade.get_session(**kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def get_backend():
|
||||||
|
"""The backend is this module itself."""
|
||||||
|
return Connection()
|
||||||
|
|
||||||
|
|
||||||
|
def model_query(model, *args, **kwargs):
|
||||||
|
"""Query helper for simpler session usage.
|
||||||
|
|
||||||
|
:param session: if present, the session to use
|
||||||
|
"""
|
||||||
|
|
||||||
|
session = kwargs.get('session') or get_session()
|
||||||
|
query = session.query(model, *args)
|
||||||
|
return query
|
||||||
|
|
||||||
|
|
||||||
|
def add_identity_filter(query, value):
|
||||||
|
"""Adds an identity filter to a query.
|
||||||
|
|
||||||
|
Filters results by ID, if supplied value is a valid integer.
|
||||||
|
Otherwise attempts to filter results by UUID.
|
||||||
|
|
||||||
|
:param query: Initial query to add filter to.
|
||||||
|
:param value: Value for filtering results by.
|
||||||
|
:return: Modified query.
|
||||||
|
"""
|
||||||
|
if strutils.is_int_like(value):
|
||||||
|
return query.filter_by(id=value)
|
||||||
|
elif uuidutils.is_uuid_like(value):
|
||||||
|
return query.filter_by(uuid=value)
|
||||||
|
else:
|
||||||
|
raise exception.InvalidIdentity(identity=value)
|
||||||
|
|
||||||
|
|
||||||
|
def _paginate_query(model, limit=None, marker=None, sort_key=None,
|
||||||
|
sort_dir=None, query=None, default_sort_key='id'):
|
||||||
|
if not query:
|
||||||
|
query = model_query(model)
|
||||||
|
sort_keys = [default_sort_key]
|
||||||
|
if sort_key and sort_key not in sort_keys:
|
||||||
|
sort_keys.insert(0, sort_key)
|
||||||
|
try:
|
||||||
|
query = db_utils.paginate_query(query, model, limit, sort_keys,
|
||||||
|
marker=marker, sort_dir=sort_dir)
|
||||||
|
except db_exc.InvalidSortKey:
|
||||||
|
raise exception.InvalidParameterValue(
|
||||||
|
_('The sort_key value "%(key)s" is an invalid field for sorting')
|
||||||
|
% {'key': sort_key})
|
||||||
|
return query.all()
|
||||||
|
|
||||||
|
|
||||||
|
class Connection(object):
|
||||||
|
"""SqlAlchemy connection."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _add_project_filters(self, context, query):
|
||||||
|
if context.is_admin and context.all_projects:
|
||||||
|
return query
|
||||||
|
|
||||||
|
if context.project_id:
|
||||||
|
query = query.filter_by(project_id=context.project_id)
|
||||||
|
else:
|
||||||
|
query = query.filter_by(user_id=context.user_id)
|
||||||
|
|
||||||
|
return query
|
||||||
|
|
||||||
|
def _add_filters(self, query, model, filters=None, filter_names=None):
|
||||||
|
"""Generic way to add filters to a Gyan model"""
|
||||||
|
if not filters:
|
||||||
|
return query
|
||||||
|
|
||||||
|
if not filter_names:
|
||||||
|
filter_names = []
|
||||||
|
|
||||||
|
for name in filter_names:
|
||||||
|
if name in filters:
|
||||||
|
value = filters[name]
|
||||||
|
if isinstance(value, list):
|
||||||
|
column = getattr(model, name)
|
||||||
|
query = query.filter(column.in_(value))
|
||||||
|
else:
|
||||||
|
column = getattr(model, name)
|
||||||
|
query = query.filter(column == value)
|
||||||
|
|
||||||
|
return query
|
||||||
|
|
||||||
|
def _add_compute_hosts_filters(self, query, filters):
|
||||||
|
filter_names = None
|
||||||
|
return self._add_filters(query, models.ComputeHost, filters=filters,
|
||||||
|
filter_names=filter_names)
|
||||||
|
|
||||||
|
def list_compute_hosts(self, context, filters=None, limit=None,
|
||||||
|
marker=None, sort_key=None, sort_dir=None):
|
||||||
|
query = model_query(models.ComputeHost)
|
||||||
|
query = self._add_compute_hosts_filters(query, filters)
|
||||||
|
return _paginate_query(models.ComputeHost, limit, marker,
|
||||||
|
sort_key, sort_dir, query,
|
||||||
|
default_sort_key='id')
|
||||||
|
|
||||||
|
def create_compute_host(self, context, values):
|
||||||
|
# ensure defaults are present for new compute hosts
|
||||||
|
if not values.get('id'):
|
||||||
|
values['id'] = uuidutils.generate_uuid()
|
||||||
|
|
||||||
|
compute_host = models.ComputeHost()
|
||||||
|
compute_host.update(values)
|
||||||
|
try:
|
||||||
|
compute_host.save()
|
||||||
|
except db_exc.DBDuplicateEntry:
|
||||||
|
raise exception.ComputeHostAlreadyExists(
|
||||||
|
field='UUID', value=values['uuid'])
|
||||||
|
return compute_host
|
||||||
|
|
||||||
|
def get_compute_host(self, context, host_uuid):
|
||||||
|
query = model_query(models.ComputeHost)
|
||||||
|
query = query.filter_by(id=host_uuid)
|
||||||
|
try:
|
||||||
|
return query.one()
|
||||||
|
except NoResultFound:
|
||||||
|
raise exception.ComputeHostNotFound(
|
||||||
|
compute_host=host_uuid)
|
||||||
|
|
||||||
|
def get_compute_host_by_hostname(self, context, hostname):
|
||||||
|
query = model_query(models.ComputeHost)
|
||||||
|
query = query.filter_by(hostname=hostname)
|
||||||
|
try:
|
||||||
|
return query.one()
|
||||||
|
except NoResultFound:
|
||||||
|
raise exception.ComputeHostNotFound(
|
||||||
|
compute_host=hostname)
|
||||||
|
except MultipleResultsFound:
|
||||||
|
raise exception.Conflict('Multiple compute hosts exist with same '
|
||||||
|
'hostname. Please use the uuid instead.')
|
||||||
|
|
||||||
|
def destroy_compute_host(self, context, host_uuid):
|
||||||
|
session = get_session()
|
||||||
|
with session.begin():
|
||||||
|
query = model_query(models.ComputeHost, session=session)
|
||||||
|
query = query.filter_by(uuid=host_uuid)
|
||||||
|
count = query.delete()
|
||||||
|
if count != 1:
|
||||||
|
raise exception.ComputeHostNotFound(
|
||||||
|
compute_host=host_uuid)
|
||||||
|
|
||||||
|
def update_compute_host(self, context, host_uuid, values):
|
||||||
|
if 'uuid' in values:
|
||||||
|
msg = _("Cannot overwrite UUID for an existing ComputeHost.")
|
||||||
|
raise exception.InvalidParameterValue(err=msg)
|
||||||
|
|
||||||
|
return self._do_update_compute_host(host_uuid, values)
|
||||||
|
|
||||||
|
def _do_update_compute_host(self, host_uuid, values):
|
||||||
|
session = get_session()
|
||||||
|
with session.begin():
|
||||||
|
query = model_query(models.ComputeHost, session=session)
|
||||||
|
query = query.filter_by(uuid=host_uuid)
|
||||||
|
try:
|
||||||
|
ref = query.with_lockmode('update').one()
|
||||||
|
except NoResultFound:
|
||||||
|
raise exception.ComputeHostNotFound(
|
||||||
|
compute_host=host_uuid)
|
||||||
|
|
||||||
|
ref.update(values)
|
||||||
|
return ref
|
||||||
|
|
||||||
|
def list_ml_models(self, context, filters=None, limit=None,
|
||||||
|
marker=None, sort_key=None, sort_dir=None):
|
||||||
|
query = model_query(models.Capsule)
|
||||||
|
query = self._add_project_filters(context, query)
|
||||||
|
query = self._add_ml_models_filters(query, filters)
|
||||||
|
return _paginate_query(models.Capsule, limit, marker,
|
||||||
|
sort_key, sort_dir, query)
|
||||||
|
|
||||||
|
def create_ml_model(self, context, values):
|
||||||
|
# ensure defaults are present for new ml_models
|
||||||
|
if not values.get('uuid'):
|
||||||
|
values['uuid'] = uuidutils.generate_uuid()
|
||||||
|
ml_model = models.ML_Model()
|
||||||
|
ml_model.update(values)
|
||||||
|
try:
|
||||||
|
ml_model.save()
|
||||||
|
except db_exc.DBDuplicateEntry:
|
||||||
|
raise exception.MLModelAlreadyExists(field='UUID',
|
||||||
|
value=values['uuid'])
|
||||||
|
return ml_model
|
||||||
|
|
||||||
|
def get_ml_model_by_uuid(self, context, ml_model_uuid):
|
||||||
|
query = model_query(models.ML_Model)
|
||||||
|
query = self._add_project_filters(context, query)
|
||||||
|
query = query.filter_by(uuid=ml_model_uuid)
|
||||||
|
try:
|
||||||
|
return query.one()
|
||||||
|
except NoResultFound:
|
||||||
|
raise exception.MLModelNotFound(ml_model=ml_model_uuid)
|
||||||
|
|
||||||
|
def get_ml_model_by_name(self, context, ml_model_name):
|
||||||
|
query = model_query(models.ML_Model)
|
||||||
|
query = self._add_project_filters(context, query)
|
||||||
|
query = query.filter_by(meta_name=ml_model_name)
|
||||||
|
try:
|
||||||
|
return query.one()
|
||||||
|
except NoResultFound:
|
||||||
|
raise exception.MLModelNotFound(ml_model=ml_model_name)
|
||||||
|
except MultipleResultsFound:
|
||||||
|
raise exception.Conflict('Multiple ml_models exist with same '
|
||||||
|
'name. Please use the ml_model uuid '
|
||||||
|
'instead.')
|
||||||
|
|
||||||
|
def destroy_ml_model(self, context, ml_model_id):
|
||||||
|
session = get_session()
|
||||||
|
with session.begin():
|
||||||
|
query = model_query(models.ML_Model, session=session)
|
||||||
|
query = add_identity_filter(query, ml_model_id)
|
||||||
|
count = query.delete()
|
||||||
|
if count != 1:
|
||||||
|
raise exception.MLModelNotFound(ml_model_id)
|
||||||
|
|
||||||
|
def update_ml_model(self, context, ml_model_id, values):
|
||||||
|
if 'uuid' in values:
|
||||||
|
msg = _("Cannot overwrite UUID for an existing ML Model.")
|
||||||
|
raise exception.InvalidParameterValue(err=msg)
|
||||||
|
|
||||||
|
return self._do_update_ml_model_id(ml_model_id, values)
|
||||||
|
|
||||||
|
def _do_update_ml_model_id(self, ml_model_id, values):
|
||||||
|
session = get_session()
|
||||||
|
with session.begin():
|
||||||
|
query = model_query(models.ML_Model, session=session)
|
||||||
|
query = add_identity_filter(query, ml_model_id)
|
||||||
|
try:
|
||||||
|
ref = query.with_lockmode('update').one()
|
||||||
|
except NoResultFound:
|
||||||
|
raise exception.MLModelNotFound(ml_model=ml_model_id)
|
||||||
|
|
||||||
|
ref.update(values)
|
||||||
|
return ref
|
||||||
|
|
||||||
|
def _add_ml_models_filters(self, query, filters):
|
||||||
|
filter_names = ['uuid', 'project_id', 'user_id']
|
||||||
|
return self._add_filters(query, models.ML_Model, filters=filters,
|
||||||
|
filter_names=filter_names)
|
||||||
111
gyan/db/sqlalchemy/migration.py
Normal file
111
gyan/db/sqlalchemy/migration.py
Normal file
@ -0,0 +1,111 @@
|
|||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
import alembic
|
||||||
|
from alembic import config as alembic_config
|
||||||
|
import alembic.migration as alembic_migration
|
||||||
|
from oslo_db import exception as db_exc
|
||||||
|
from oslo_db.sqlalchemy import enginefacade
|
||||||
|
from oslo_db.sqlalchemy.migration_cli import manager
|
||||||
|
|
||||||
|
from gyan.db.sqlalchemy import models
|
||||||
|
|
||||||
|
import gyan.conf
|
||||||
|
|
||||||
|
_MANAGER = None
|
||||||
|
|
||||||
|
|
||||||
|
def _alembic_config():
|
||||||
|
path = os.path.join(os.path.dirname(__file__), 'alembic.ini')
|
||||||
|
config = alembic_config.Config(path)
|
||||||
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
def get_manager():
|
||||||
|
global _MANAGER
|
||||||
|
if not _MANAGER:
|
||||||
|
alembic_path = os.path.abspath(
|
||||||
|
os.path.join(os.path.dirname(__file__), 'alembic.ini'))
|
||||||
|
migrate_path = os.path.abspath(
|
||||||
|
os.path.join(os.path.dirname(__file__), 'alembic'))
|
||||||
|
migration_config = {'alembic_ini_path': alembic_path,
|
||||||
|
'alembic_repo_path': migrate_path,
|
||||||
|
'db_url': gyan.conf.CONF.database.connection}
|
||||||
|
_MANAGER = manager.MigrationManager(migration_config)
|
||||||
|
|
||||||
|
return _MANAGER
|
||||||
|
|
||||||
|
|
||||||
|
def version(config=None, engine=None):
|
||||||
|
"""Current database version.
|
||||||
|
|
||||||
|
:returns: Database version
|
||||||
|
:rtype: string
|
||||||
|
"""
|
||||||
|
if engine is None:
|
||||||
|
engine = enginefacade.get_legacy_facade().get_engine()
|
||||||
|
with engine.connect() as conn:
|
||||||
|
context = alembic_migration.MigrationContext.configure(conn)
|
||||||
|
return context.get_current_revision()
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade(version):
|
||||||
|
"""Used for upgrading database.
|
||||||
|
|
||||||
|
:param version: Desired database version
|
||||||
|
:type version: string
|
||||||
|
"""
|
||||||
|
version = version or 'head'
|
||||||
|
|
||||||
|
get_manager().upgrade(version)
|
||||||
|
|
||||||
|
|
||||||
|
def stamp(revision, config=None):
|
||||||
|
"""Stamps database with provided revision.
|
||||||
|
|
||||||
|
Don't run any migrations.
|
||||||
|
|
||||||
|
:param revision: Should match one from repository or head - to stamp
|
||||||
|
database with most recent revision
|
||||||
|
:type revision: string
|
||||||
|
"""
|
||||||
|
config = config or _alembic_config()
|
||||||
|
return alembic.command.stamp(config, revision=revision)
|
||||||
|
|
||||||
|
|
||||||
|
def create_schema(config=None, engine=None):
|
||||||
|
"""Create database schema from models description.
|
||||||
|
|
||||||
|
Can be used for initial installation instead of upgrade('head').
|
||||||
|
"""
|
||||||
|
if engine is None:
|
||||||
|
engine = enginefacade.get_legacy_facade().get_engine()
|
||||||
|
|
||||||
|
if version(engine=engine) is not None:
|
||||||
|
raise db_exc.DBMigrationError("DB schema is already under version"
|
||||||
|
" control. Use upgrade() instead")
|
||||||
|
models.Base.metadata.create_all(engine)
|
||||||
|
stamp('head', config=config)
|
||||||
|
|
||||||
|
|
||||||
|
def revision(message=None, autogenerate=False):
|
||||||
|
"""Creates template for migration.
|
||||||
|
|
||||||
|
:param message: Text that will be used for migration title
|
||||||
|
:type message: string
|
||||||
|
:param autogenerate: If True - generates diff based on current database
|
||||||
|
state
|
||||||
|
:type autogenerate: bool
|
||||||
|
"""
|
||||||
|
return get_manager().revision(message=message, autogenerate=autogenerate)
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user