Browse Source

[WIP] Add hosts APIs and compute rpc APIs

Change-Id: I5893bf066fc876e795f020214080fb92b79a00a5
bharath 6 months ago
parent
commit
a40f556027
100 changed files with 6821 additions and 0 deletions
  1. 9
    0
      devstack/README.rst
  2. 325
    0
      devstack/lib/gyan
  3. 13
    0
      devstack/local.conf.sample
  4. 18
    0
      devstack/local.conf.subnode.sample
  5. 47
    0
      devstack/plugin.sh
  6. 24
    0
      devstack/settings
  7. 41
    0
      etc/apache2/gyan.conf.template
  8. 19
    0
      etc/gyan/api-paste.ini
  9. 14
    0
      etc/gyan/gyan-config-generator.conf
  10. 3
    0
      etc/gyan/gyan-policy-generator.conf
  11. 27
    0
      etc/gyan/rootwrap.conf
  12. 8
    0
      etc/gyan/rootwrap.d/gyan.filters
  13. 1
    0
      gyan/MANIFEST.in
  14. 0
    0
      gyan/api/__init__.py
  15. 67
    0
      gyan/api/app.py
  16. 19
    0
      gyan/api/app.wsgi
  17. 25
    0
      gyan/api/config.py
  18. 0
    0
      gyan/api/controllers/__init__.py
  19. 233
    0
      gyan/api/controllers/base.py
  20. 35
    0
      gyan/api/controllers/link.py
  21. 97
    0
      gyan/api/controllers/root.py
  22. 155
    0
      gyan/api/controllers/v1/__init__.py
  23. 41
    0
      gyan/api/controllers/v1/collection.py
  24. 109
    0
      gyan/api/controllers/v1/hosts.py
  25. 289
    0
      gyan/api/controllers/v1/ml_models.py
  26. 0
    0
      gyan/api/controllers/v1/schemas/__init__.py
  27. 49
    0
      gyan/api/controllers/v1/schemas/ml_models.py
  28. 97
    0
      gyan/api/controllers/v1/schemas/parameter_types.py
  29. 50
    0
      gyan/api/controllers/v1/schemas/services.py
  30. 0
    0
      gyan/api/controllers/v1/views/__init__.py
  31. 43
    0
      gyan/api/controllers/v1/views/hosts_view.py
  32. 55
    0
      gyan/api/controllers/v1/views/ml_models_view.py
  33. 145
    0
      gyan/api/controllers/versions.py
  34. 114
    0
      gyan/api/hooks.py
  35. 69
    0
      gyan/api/http_error.py
  36. 21
    0
      gyan/api/middleware/__init__.py
  37. 71
    0
      gyan/api/middleware/auth_token.py
  38. 99
    0
      gyan/api/middleware/parsable_error.py
  39. 37
    0
      gyan/api/servicegroup.py
  40. 116
    0
      gyan/api/utils.py
  41. 57
    0
      gyan/api/validation/__init__.py
  42. 80
    0
      gyan/api/validation/validators.py
  43. 33
    0
      gyan/api/versioned_method.py
  44. 36
    0
      gyan/api/wsgi.py
  45. 17
    0
      gyan/cmd/__init__.py
  46. 45
    0
      gyan/cmd/api.py
  47. 47
    0
      gyan/cmd/compute.py
  48. 67
    0
      gyan/cmd/db_manage.py
  49. 0
    0
      gyan/common/__init__.py
  50. 54
    0
      gyan/common/config.py
  51. 17
    0
      gyan/common/consts.py
  52. 180
    0
      gyan/common/context.py
  53. 485
    0
      gyan/common/exception.py
  54. 24
    0
      gyan/common/i18n.py
  55. 89
    0
      gyan/common/keystone.py
  56. 47
    0
      gyan/common/paths.py
  57. 24
    0
      gyan/common/policies/__init__.py
  58. 41
    0
      gyan/common/policies/base.py
  59. 46
    0
      gyan/common/policies/host.py
  60. 123
    0
      gyan/common/policies/ml_model.py
  61. 155
    0
      gyan/common/policy.py
  62. 22
    0
      gyan/common/privileged.py
  63. 101
    0
      gyan/common/profiler.py
  64. 120
    0
      gyan/common/rpc.py
  65. 105
    0
      gyan/common/rpc_service.py
  66. 92
    0
      gyan/common/service.py
  67. 63
    0
      gyan/common/short_id.py
  68. 25
    0
      gyan/common/singleton.py
  69. 255
    0
      gyan/common/utils.py
  70. 33
    0
      gyan/common/yamlutils.py
  71. 0
    0
      gyan/compute/__init__.py
  72. 63
    0
      gyan/compute/api.py
  73. 67
    0
      gyan/compute/compute_host_tracker.py
  74. 121
    0
      gyan/compute/manager.py
  75. 68
    0
      gyan/compute/rpcapi.py
  76. 41
    0
      gyan/conf/__init__.py
  77. 67
    0
      gyan/conf/api.py
  78. 40
    0
      gyan/conf/compute.py
  79. 32
    0
      gyan/conf/database.py
  80. 51
    0
      gyan/conf/gyan_client.py
  81. 35
    0
      gyan/conf/keystone.py
  82. 47
    0
      gyan/conf/ml_model_driver.py
  83. 76
    0
      gyan/conf/opts.py
  84. 42
    0
      gyan/conf/path.py
  85. 29
    0
      gyan/conf/profiler.py
  86. 104
    0
      gyan/conf/scheduler.py
  87. 36
    0
      gyan/conf/services.py
  88. 27
    0
      gyan/conf/ssl.py
  89. 31
    0
      gyan/conf/utils.py
  90. 21
    0
      gyan/db/__init__.py
  91. 197
    0
      gyan/db/api.py
  92. 47
    0
      gyan/db/migration.py
  93. 0
    0
      gyan/db/sqlalchemy/__init__.py
  94. 68
    0
      gyan/db/sqlalchemy/alembic.ini
  95. 11
    0
      gyan/db/sqlalchemy/alembic/README
  96. 58
    0
      gyan/db/sqlalchemy/alembic/env.py
  97. 20
    0
      gyan/db/sqlalchemy/alembic/script.py.mako
  98. 48
    0
      gyan/db/sqlalchemy/alembic/versions/cebd81b206ca_add_model_and_host_tables.py
  99. 306
    0
      gyan/db/sqlalchemy/api.py
  100. 0
    0
      gyan/db/sqlalchemy/migration.py

+ 9
- 0
devstack/README.rst View File

@@ -0,0 +1,9 @@
1
+====================
2
+DevStack Integration
3
+====================
4
+
5
+This directory contains the files necessary to integrate gyan with devstack.
6
+
7
+Refer the quickstart guide at
8
+https://docs.openstack.org/gyan/latest/contributor/quickstart.html
9
+for more information on using devstack and gyan.

+ 325
- 0
devstack/lib/gyan View File

@@ -0,0 +1,325 @@
1
+#!/bin/bash
2
+#
3
+# lib/gyan
4
+# Functions to control the configuration and operation of the **gyan** service
5
+
6
+# Dependencies:
7
+#
8
+# - ``functions`` file
9
+# - ``DEST``, ``DATA_DIR``, ``STACK_USER`` must be defined
10
+# - ``SERVICE_{TENANT_NAME|PASSWORD}`` must be defined
11
+
12
+# ``stack.sh`` calls the entry points in this order:
13
+#
14
+# - install_gyan
15
+# - configure_gyan
16
+# - create_gyan_conf
17
+# - create_gyan_accounts
18
+# - init_gyan
19
+# - start_gyan
20
+# - stop_gyan
21
+# - cleanup_gyan
22
+
23
+# Save trace setting
24
+XTRACE=$(set +o | grep xtrace)
25
+set +o xtrace
26
+
27
+
28
+# Defaults
29
+# --------
30
+
31
+# Set up default directories
32
+GYAN_REPO=${GYAN_REPO:-${GIT_BASE}/openstack/gyan.git}
33
+GYAN_BRANCH=${GYAN_BRANCH:-master}
34
+GYAN_DIR=$DEST/gyan
35
+
36
+GITREPO["python-gyanclient"]=${GYANCLIENT_REPO:-${GIT_BASE}/openstack/python-gyanclient.git}
37
+GITBRANCH["python-gyanclient"]=${GYANCLIENT_BRANCH:-master}
38
+GITDIR["python-gyanclient"]=$DEST/python-gyanclient
39
+
40
+GYAN_STATE_PATH=${GYAN_STATE_PATH:=$DATA_DIR/gyan}
41
+GYAN_AUTH_CACHE_DIR=${GYAN_AUTH_CACHE_DIR:-/var/cache/gyan}
42
+
43
+GYAN_CONF_DIR=/etc/gyan
44
+GYAN_CONF=$GYAN_CONF_DIR/gyan.conf
45
+GYAN_API_PASTE=$GYAN_CONF_DIR/api-paste.ini
46
+
47
+if is_ssl_enabled_service "gyan" || is_service_enabled tls-proxy; then
48
+    GYAN_SERVICE_PROTOCOL="https"
49
+fi
50
+
51
+# Toggle for deploying GYAN-API under a wsgi server
52
+GYAN_USE_UWSGI=${GYAN_USE_UWSGI:-True}
53
+
54
+
55
+# Public facing bits
56
+GYAN_SERVICE_HOST=${GYAN_SERVICE_HOST:-$SERVICE_HOST}
57
+GYAN_SERVICE_PORT=${GYAN_SERVICE_PORT:-8517}
58
+GYAN_SERVICE_PORT_INT=${GYAN_SERVICE_PORT_INT:-18517}
59
+GYAN_SERVICE_PROTOCOL=${GYAN_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
60
+
61
+GYAN_TRUSTEE_DOMAIN_ADMIN_PASSWORD=${GYAN_TRUSTEE_DOMAIN_ADMIN_PASSWORD:-secret}
62
+
63
+# Support entry points installation of console scripts
64
+if [[ -d $GYAN_DIR/bin ]]; then
65
+    GYAN_BIN_DIR=$GYAN_DIR/bin
66
+else
67
+    GYAN_BIN_DIR=$(get_python_exec_prefix)
68
+fi
69
+
70
+GYAN_UWSGI=$GYAN_BIN_DIR/gyan-api-wsgi
71
+GYAN_UWSGI_CONF=$GYAN_CONF_DIR/gyan-api-uwsgi.ini
72
+
73
+GYAN_DB_TYPE=${GYAN_DB_TYPE:-sql}
74
+
75
+if is_ubuntu; then
76
+    UBUNTU_RELEASE_BASE_NUM=`lsb_release -r | awk '{print $2}' | cut -d '.' -f 1`
77
+fi
78
+
79
+# Functions
80
+# ---------
81
+
82
+# cleanup_gyan() - Remove residual data files, anything left over from previous
83
+# runs that a clean run would need to clean up
84
+function cleanup_gyan {
85
+    sudo rm -rf $GYAN_STATE_PATH $GYAN_AUTH_CACHE_DIR
86
+
87
+    remove_uwsgi_config "$GYAN_UWSGI_CONF" "$GYAN_UWSGI"
88
+}
89
+
90
+# configure_gyan() - Set config files, create data dirs, etc
91
+function configure_gyan {
92
+    # Put config files in ``/etc/gyan`` for everyone to find
93
+    if [[ ! -d $GYAN_CONF_DIR ]]; then
94
+        sudo mkdir -p $GYAN_CONF_DIR
95
+        sudo chown $STACK_USER $GYAN_CONF_DIR
96
+    fi
97
+
98
+    configure_rootwrap gyan
99
+
100
+    # Rebuild the config file from scratch
101
+    create_gyan_conf
102
+
103
+    create_api_paste_conf
104
+
105
+    write_uwsgi_config "$GYAN_UWSGI_CONF" "$GYAN_UWSGI" "/ml-infra"
106
+
107
+    if [[ "$USE_PYTHON3" = "True" ]]; then
108
+        # Switch off glance->swift communication as swift fails under py3.x
109
+        iniset /etc/glance/glance-api.conf glance_store default_store file
110
+    fi
111
+}
112
+
113
+
114
+# create_gyan_accounts() - Set up common required GYAN accounts
115
+#
116
+# Project              User         Roles
117
+# ------------------------------------------------------------------
118
+# SERVICE_PROJECT_NAME  gyan         service
119
+function create_gyan_accounts {
120
+
121
+    create_service_user "gyan" "admin"
122
+
123
+    if is_service_enabled gyan-api; then
124
+
125
+        local gyan_api_url
126
+        if [[ "$GYAN_USE_UWSGI" == "True" ]]; then
127
+            gyan_api_url="$GYAN_SERVICE_PROTOCOL://$GYAN_SERVICE_HOST/ml-infra"
128
+        else
129
+            gyan_api_url="$GYAN_SERVICE_PROTOCOL://$GYAN_SERVICE_HOST:$GYAN_SERVICE_PORT"
130
+        fi
131
+
132
+        local gyan_service=$(get_or_create_service "gyan" \
133
+            "ml-infra" "ML Infra As Service")
134
+        get_or_create_endpoint $gyan_service \
135
+            "$REGION_NAME" \
136
+            "$gyan_api_url/v1" \
137
+            "$gyan_api_url/v1" \
138
+            "$gyan_api_url/v1"
139
+    fi
140
+
141
+}
142
+
143
+# create_gyan_conf() - Create a new gyan.conf file
144
+function create_gyan_conf {
145
+
146
+    # (Re)create ``gyan.conf``
147
+    rm -f $GYAN_CONF
148
+    if [[ ${GYAN_DRIVER} == "tensorflow" ]]; then
149
+        iniset $GYAN_CONF DEFAULT ml_model_driver "ml_model.driver.TensorflowDriver"
150
+    fi
151
+    if [[ ${GYAN_DB_TYPE} == "sql" ]]; then
152
+        iniset $GYAN_CONF DEFAULT db_type sql
153
+    fi
154
+    iniset $GYAN_CONF DEFAULT debug "$ENABLE_DEBUG_LOG_LEVEL"
155
+    iniset $GYAN_CONF DEFAULT my_ip "$HOST_IP"
156
+    iniset $GYAN_CONF DEFAULT host  "$HOST_IP"
157
+    iniset $GYAN_CONF oslo_messaging_rabbit rabbit_userid $RABBIT_USERID
158
+    iniset $GYAN_CONF oslo_messaging_rabbit rabbit_password $RABBIT_PASSWORD
159
+    iniset $GYAN_CONF oslo_messaging_rabbit rabbit_host $RABBIT_HOST
160
+    iniset $GYAN_CONF database connection `database_connection_url gyan`
161
+    iniset $GYAN_CONF api host_ip "$GYAN_SERVICE_HOST"
162
+    iniset $GYAN_CONF api port "$GYAN_SERVICE_PORT"
163
+
164
+    iniset $GYAN_CONF keystone_auth auth_type password
165
+    iniset $GYAN_CONF keystone_auth username gyan
166
+    iniset $GYAN_CONF keystone_auth password $SERVICE_PASSWORD
167
+    iniset $GYAN_CONF keystone_auth project_name $SERVICE_PROJECT_NAME
168
+    iniset $GYAN_CONF keystone_auth project_domain_id default
169
+    iniset $GYAN_CONF keystone_auth user_domain_id default
170
+
171
+    # FIXME(pauloewerton): keystone_authtoken section is deprecated. Remove it
172
+    # after deprecation period.
173
+    iniset $GYAN_CONF keystone_authtoken admin_user gyan
174
+    iniset $GYAN_CONF keystone_authtoken admin_password $SERVICE_PASSWORD
175
+    iniset $GYAN_CONF keystone_authtoken admin_tenant_name $SERVICE_PROJECT_NAME
176
+
177
+    configure_auth_token_middleware $GYAN_CONF gyan $GYAN_AUTH_CACHE_DIR
178
+
179
+    iniset $GYAN_CONF keystone_auth auth_url $KEYSTONE_AUTH_URI_V3
180
+    iniset $GYAN_CONF keystone_authtoken www_authenticate_uri $KEYSTONE_SERVICE_URI_V3
181
+    iniset $GYAN_CONF keystone_authtoken auth_url $KEYSTONE_AUTH_URI_V3
182
+    iniset $GYAN_CONF keystone_authtoken auth_version v3
183
+
184
+
185
+    if is_fedora || is_suse; then
186
+        # gyan defaults to /usr/local/bin, but fedora and suse pip like to
187
+        # install things in /usr/bin
188
+        iniset $GYAN_CONF DEFAULT bindir "/usr/bin"
189
+    fi
190
+
191
+    if [ -n "$GYAN_STATE_PATH" ]; then
192
+        iniset $GYAN_CONF DEFAULT state_path "$GYAN_STATE_PATH"
193
+        iniset $GYAN_CONF oslo_concurrency lock_path "$GYAN_STATE_PATH"
194
+    fi
195
+
196
+    if [ "$SYSLOG" != "False" ]; then
197
+        iniset $GYAN_CONF DEFAULT use_syslog "True"
198
+    fi
199
+
200
+    # Format logging
201
+    if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then
202
+        setup_colorized_logging $GYAN_CONF DEFAULT
203
+    else
204
+        # Show user_name and project_name instead of user_id and project_id
205
+        iniset $GYAN_CONF DEFAULT logging_context_format_string "%(asctime)s.%(msecs)03d %(levelname)s %(name)s [%(request_id)s %(user_name)s %(project_name)s] %(instance)s%(message)s"
206
+    fi
207
+
208
+    # Register SSL certificates if provided
209
+    if is_ssl_enabled_service gyan; then
210
+        ensure_certificates gyan
211
+
212
+        iniset $GYAN_CONF DEFAULT ssl_cert_file "$GYAN_SSL_CERT"
213
+        iniset $GYAN_CONF DEFAULT ssl_key_file "$GYAN_SSL_KEY"
214
+
215
+        iniset $GYAN_CONF DEFAULT enabled_ssl_apis "$GYAN_ENABLED_APIS"
216
+    fi
217
+}
218
+
219
+function create_api_paste_conf {
220
+    # copy api_paste.ini
221
+    cp $GYAN_DIR/etc/gyan/api-paste.ini $GYAN_API_PASTE
222
+}
223
+
224
+# create_gyan_cache_dir() - Part of the init_GYAN() process
225
+function create_gyan_cache_dir {
226
+    # Create cache dir
227
+    sudo mkdir -p $GYAN_AUTH_CACHE_DIR
228
+    sudo chown $STACK_USER $GYAN_AUTH_CACHE_DIR
229
+    rm -f $GYAN_AUTH_CACHE_DIR/*
230
+}
231
+
232
+
233
+# init_gyan() - Initialize databases, etc.
234
+function init_gyan {
235
+    # Only do this step once on the API node for an entire cluster.
236
+    if is_service_enabled gyan-api; then
237
+        if is_service_enabled $DATABASE_BACKENDS; then
238
+            # (Re)create gyan database
239
+            recreate_database gyan
240
+
241
+            # Migrate gyan database
242
+            $GYAN_BIN_DIR/gyan-db-manage upgrade
243
+        fi
244
+
245
+        if is_service_enabled gyan-etcd; then
246
+            install_etcd_server
247
+        fi
248
+        create_gyan_cache_dir
249
+    fi
250
+}
251
+
252
+# install_gyanclient() - Collect source and prepare
253
+function install_gyanclient {
254
+    if use_library_from_git "python-gyanclient"; then
255
+        git_clone_by_name "python-gyanclient"
256
+        setup_dev_lib "python-gyanclient"
257
+        sudo install -D -m 0644 -o $STACK_USER {${GITDIR["python-gyanclient"]}/tools/,/etc/bash_completion.d/}gyan.bash_completion
258
+    fi
259
+}
260
+
261
+# install_gyan() - Collect source and prepare
262
+function install_gyan {
263
+    git_clone $GYAN_REPO $GYAN_DIR $GYAN_BRANCH
264
+    setup_develop $GYAN_DIR
265
+}
266
+
267
+# start_gyan_api() - Start the API process ahead of other things
268
+function start_gyan_api {
269
+    # Get right service port for testing
270
+    local service_port=$GYAN_SERVICE_PORT
271
+    local service_protocol=$GYAN_SERVICE_PROTOCOL
272
+    if is_service_enabled tls-proxy; then
273
+        service_port=$GYAN_SERVICE_PORT_INT
274
+        service_protocol="http"
275
+    fi
276
+
277
+    local gyan_url
278
+    if [ "$GYAN_USE_UWSGI" == "True" ]; then
279
+        run_process gyan-api "$GYAN_BIN_DIR/uwsgi --procname-prefix gyan-api --ini $GYAN_UWSGI_CONF"
280
+        gyan_url=$service_protocol://$GYAN_SERVICE_HOST/ml-infra
281
+    else
282
+        run_process gyan-api "$GYAN_BIN_DIR/gyan-api"
283
+        gyan_url=$service_protocol://$GYAN_SERVICE_HOST:$service_port
284
+    fi
285
+
286
+    echo "Waiting for gyan-api to start..."
287
+    if ! wait_for_service $SERVICE_TIMEOUT $gyan_url; then
288
+        die $LINENO "gyan-api did not start"
289
+    fi
290
+
291
+    # Start proxies if enabled
292
+    if is_service_enabled tls-proxy; then
293
+        start_tls_proxy '*' $GYAN_SERVICE_PORT $GYAN_SERVICE_HOST $GYAN_SERVICE_PORT_INT &
294
+    fi
295
+}
296
+
297
+# start_gyan_compute() - Start Gyan compute agent
298
+function start_gyan_compute {
299
+    echo "Start gyan compute..."
300
+    run_process gyan-compute "$GYAN_BIN_DIR/gyan-compute"
301
+}
302
+
303
+
304
+# start_gyan() - Start running processes, including screen
305
+function start_gyan {
306
+
307
+    # ``run_process`` checks ``is_service_enabled``, it is not needed here
308
+    start_gyan_api
309
+    start_gyan_compute
310
+}
311
+
312
+# stop_gyan() - Stop running processes (non-screen)
313
+function stop_gyan {
314
+
315
+    if [ "$GYAN_USE_UWSGI" == "True" ]; then
316
+        disable_apache_site gyan
317
+        restart_apache_server
318
+    else
319
+        stop_process gyan-api
320
+    fi
321
+    stop_process gyan-compute
322
+}
323
+
324
+# Restore xtrace
325
+$XTRACE

+ 13
- 0
devstack/local.conf.sample View File

@@ -0,0 +1,13 @@
1
+[[local|localrc]]
2
+HOST_IP=10.0.0.11 # change this to your IP address
3
+DATABASE_PASSWORD=password
4
+RABBIT_PASSWORD=password
5
+SERVICE_TOKEN=password
6
+SERVICE_PASSWORD=password
7
+ADMIN_PASSWORD=password
8
+enable_plugin gyan https://git.openstack.org/openstack/gyan
9
+
10
+# install python-gyanclient from git
11
+LIBS_FROM_GIT="python-gyanclient"
12
+
13
+

+ 18
- 0
devstack/local.conf.subnode.sample View File

@@ -0,0 +1,18 @@
1
+[[local|localrc]]
2
+HOST_IP=10.0.0.31 # change this to your IP address
3
+DATABASE_PASSWORD=password
4
+RABBIT_PASSWORD=password
5
+SERVICE_TOKEN=password
6
+SERVICE_PASSWORD=password
7
+ADMIN_PASSWORD=password
8
+enable_plugin gyan https://git.openstack.org/openstack/gyan
9
+
10
+
11
+# Following is for multi host settings
12
+MULTI_HOST=True
13
+SERVICE_HOST=10.0.0.11 # change this to controller's IP address
14
+DATABASE_TYPE=mysql
15
+MYSQL_HOST=$SERVICE_HOST
16
+RABBIT_HOST=$SERVICE_HOST
17
+
18
+ENABLED_SERVICES=gyan-compute

+ 47
- 0
devstack/plugin.sh View File

@@ -0,0 +1,47 @@
1
+# gyan - Devstack extras script to install gyan
2
+
3
+# Save trace setting
4
+XTRACE=$(set +o | grep xtrace)
5
+set -o xtrace
6
+
7
+echo_summary "gyan's plugin.sh was called..."
8
+source $DEST/gyan/devstack/lib/gyan
9
+(set -o posix; set)
10
+
11
+if is_service_enabled gyan-api gyan-compute; then
12
+    if [[ "$1" == "stack" && "$2" == "install" ]]; then
13
+        echo_summary "Installing gyan"
14
+        install_gyan
15
+
16
+        install_gyanclient
17
+        cleanup_gyan
18
+
19
+    elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
20
+        echo_summary "Configuring gyan"
21
+        configure_gyan
22
+
23
+        if is_service_enabled key; then
24
+            create_gyan_accounts
25
+        fi
26
+
27
+    elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
28
+        # Initialize gyan
29
+        init_gyan
30
+
31
+        # Start the gyan API and gyan compute
32
+        echo_summary "Starting gyan"
33
+        start_gyan
34
+
35
+    fi
36
+
37
+    if [[ "$1" == "unstack" ]]; then
38
+        stop_gyan
39
+    fi
40
+
41
+    if [[ "$1" == "clean" ]]; then
42
+        cleanup_gyan
43
+    fi
44
+fi
45
+
46
+# Restore xtrace
47
+$XTRACE

+ 24
- 0
devstack/settings View File

@@ -0,0 +1,24 @@
1
+# Devstack settings
2
+
3
+## Modify to your environment
4
+# FLOATING_RANGE=192.168.1.224/27
5
+# PUBLIC_NETWORK_GATEWAY=192.168.1.225
6
+# PUBLIC_INTERFACE=em1
7
+# FIXED_RANGE=10.0.0.0/24
8
+## Log all output to files
9
+# LOGFILE=$HOME/devstack.log
10
+## Neutron settings
11
+# Q_USE_SECGROUP=True
12
+# ENABLE_TENANT_VLANS=True
13
+# TENANT_VLAN_RANGE=
14
+# PHYSICAL_NETWORK=public
15
+# OVS_PHYSICAL_BRIDGE=br-ex
16
+
17
+# Enable Gyan services
18
+if [[ ${HOST_IP} == ${SERVICE_HOST} ]]; then
19
+    enable_service gyan-api
20
+    enable_service gyan-compute
21
+else
22
+    enable_service gyan-compute
23
+fi
24
+

+ 41
- 0
etc/apache2/gyan.conf.template View File

@@ -0,0 +1,41 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+# not use this file except in compliance with the License. You may obtain
3
+# a copy of the License at
4
+#
5
+#      http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+# License for the specific language governing permissions and limitations
11
+# under the License.
12
+
13
+# This is an example Apache2 configuration file for using the
14
+# gyan API through mod_wsgi.
15
+
16
+# Note: If you are using a Debian-based system then the paths
17
+# "/var/log/httpd" and "/var/run/httpd" will use "apache2" instead
18
+# of "httpd".
19
+#
20
+# The number of processes and threads is an example only and should
21
+# be adjusted according to local requirements.
22
+
23
+Listen %PUBLICPORT%
24
+LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D(us)" gyan_combined
25
+
26
+<VirtualHost *:%PUBLICPORT%>
27
+    WSGIDaemonProcess gyan-api user=%USER% processes=5 threads=1 display-name=%{GROUP}
28
+    WSGIScriptAlias / %PUBLICWSGI%
29
+    WSGIProcessGroup gyan-api
30
+    ErrorLogFormat "%M"
31
+    ErrorLog /var/log/%APACHE_NAME%/gyan_api.log
32
+    LogLevel info
33
+    CustomLog /var/log/%APACHE_NAME%/gyan_access.log gyan_combined
34
+
35
+    <Directory /opt/stack/gyan/gyan/api>
36
+        WSGIProcessGroup gyan-api
37
+        WSGIApplicationGroup %{GLOBAL}
38
+        AllowOverride All
39
+        Require all granted
40
+    </Directory>
41
+</VirtualHost>

+ 19
- 0
etc/gyan/api-paste.ini View File

@@ -0,0 +1,19 @@
1
+[pipeline:main]
2
+pipeline = cors request_id osprofiler authtoken api_v1
3
+
4
+[app:api_v1]
5
+paste.app_factory = gyan.api.app:app_factory
6
+
7
+[filter:authtoken]
8
+acl_public_routes = /, /v1
9
+paste.filter_factory = gyan.api.middleware.auth_token:AuthTokenMiddleware.factory
10
+
11
+[filter:osprofiler]
12
+paste.filter_factory = gyan.common.profiler:WsgiMiddleware.factory
13
+
14
+[filter:request_id]
15
+paste.filter_factory = oslo_middleware:RequestId.factory
16
+
17
+[filter:cors]
18
+paste.filter_factory =  oslo_middleware.cors:filter_factory
19
+oslo_config_project = gyan

+ 14
- 0
etc/gyan/gyan-config-generator.conf View File

@@ -0,0 +1,14 @@
1
+[DEFAULT]
2
+output_file = etc/gyan/gyan.conf.sample
3
+wrap_width = 79
4
+
5
+namespace = gyan.conf
6
+namespace = keystonemiddleware.auth_token
7
+namespace = oslo.concurrency
8
+namespace = oslo.db
9
+namespace = oslo.log
10
+namespace = oslo.messaging
11
+namespace = oslo.middleware.cors
12
+namespace = oslo.policy
13
+namespace = oslo.service.periodic_task
14
+namespace = oslo.service.service

+ 3
- 0
etc/gyan/gyan-policy-generator.conf View File

@@ -0,0 +1,3 @@
1
+[DEFAULT]
2
+output_file = etc/gyan/policy.yaml.sample
3
+namespace = gyan

+ 27
- 0
etc/gyan/rootwrap.conf View File

@@ -0,0 +1,27 @@
1
+# Configuration for gyan-rootwrap
2
+# This file should be owned by (and only-writable by) the root user
3
+
4
+[DEFAULT]
5
+# List of directories to load filter definitions from (separated by ',').
6
+# These directories MUST all be only writable by root !
7
+filters_path=/etc/gyan/rootwrap.d
8
+
9
+# List of directories to search executables in, in case filters do not
10
+# explicitely specify a full path (separated by ',')
11
+# If not specified, defaults to system PATH environment variable.
12
+# These directories MUST all be only writable by root !
13
+exec_dirs=/sbin,/usr/sbin,/bin,/usr/bin,/usr/local/bin,/usr/local/sbin
14
+
15
+# Enable logging to syslog
16
+# Default value is False
17
+use_syslog=False
18
+
19
+# Which syslog facility to use.
20
+# Valid values include auth, authpriv, syslog, local0, local1...
21
+# Default value is 'syslog'
22
+syslog_log_facility=syslog
23
+
24
+# Which messages to log.
25
+# INFO means log all usage
26
+# ERROR means only log unsuccessful attempts
27
+syslog_log_level=ERROR

+ 8
- 0
etc/gyan/rootwrap.d/gyan.filters View File

@@ -0,0 +1,8 @@
1
+# gyan command filters
2
+# This file should be owned by (and only-writeable by) the root user
3
+
4
+[Filters]
5
+# privileged/__init__.py: priv_context.PrivContext(default)
6
+# This line ties the superuser privs with the config files, context name,
7
+# and (implicitly) the actual python code invoked.
8
+privsep-rootwrap: RegExpFilter, privsep-helper, root, privsep-helper, --config-file, /etc/(?!\.\.).*, --privsep_context, os_brick.privileged.default, --privsep_sock_path, /tmp/.*

+ 1
- 0
gyan/MANIFEST.in View File

@@ -0,0 +1 @@
1
+recursive-include public *

+ 0
- 0
gyan/api/__init__.py View File


+ 67
- 0
gyan/api/app.py View File

@@ -0,0 +1,67 @@
1
+#    Licensed under the Apache License, Version 2.0 (the "License");
2
+#    you may not use this file except in compliance with the License.
3
+#    You may obtain a copy of the License at
4
+#
5
+#        http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+#    Unless required by applicable law or agreed to in writing, software
8
+#    distributed under the License is distributed on an "AS IS" BASIS,
9
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10
+#    See the License for the specific language governing permissions and
11
+#    limitations under the License.
12
+
13
+import os
14
+
15
+from oslo_config import cfg
16
+from oslo_log import log
17
+from paste import deploy
18
+import pecan
19
+
20
+from gyan.api import config as api_config
21
+from gyan.api import middleware
22
+from gyan.common import config as common_config
23
+import gyan.conf
24
+
25
+CONF = gyan.conf.CONF
26
+LOG = log.getLogger(__name__)
27
+
28
+
29
+def get_pecan_config():
30
+    # Set up the pecan configuration
31
+    filename = api_config.__file__.replace('.pyc', '.py')
32
+    return pecan.configuration.conf_from_file(filename)
33
+
34
+
35
+def setup_app(config=None):
36
+    if not config:
37
+        config = get_pecan_config()
38
+
39
+    app_conf = dict(config.app)
40
+    common_config.set_config_defaults()
41
+
42
+    app = pecan.make_app(
43
+        app_conf.pop('root'),
44
+        logging=getattr(config, 'logging', {}),
45
+        wrap_app=middleware.ParsableErrorMiddleware,
46
+        **app_conf
47
+    )
48
+
49
+    return app
50
+
51
+
52
+def load_app():
53
+    cfg_file = None
54
+    cfg_path = CONF.api.api_paste_config
55
+    if not os.path.isabs(cfg_path):
56
+        cfg_file = CONF.find_file(cfg_path)
57
+    elif os.path.exists(cfg_path):
58
+        cfg_file = cfg_path
59
+
60
+    if not cfg_file:
61
+        raise cfg.ConfigFilesNotFoundError([CONF.api.api_paste_config])
62
+    LOG.info("Full WSGI config used: %s", cfg_file)
63
+    return deploy.loadapp("config:" + cfg_file)
64
+
65
+
66
+def app_factory(global_config, **local_conf):
67
+    return setup_app()

+ 19
- 0
gyan/api/app.wsgi View File

@@ -0,0 +1,19 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+# not use this file except in compliance with the License. You may obtain
3
+# a copy of the License at
4
+#
5
+#      http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+# License for the specific language governing permissions and limitations
11
+# under the License.
12
+
13
+"""Use this file for deploying the API under mod_wsgi.
14
+See https://pecan.readthedocs.org/en/latest/deployment.html for details.
15
+"""
16
+
17
+from gyan.api import wsgi
18
+
19
+application = wsgi.init_application()

+ 25
- 0
gyan/api/config.py View File

@@ -0,0 +1,25 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+# not use this file except in compliance with the License. You may obtain
3
+# a copy of the License at
4
+#
5
+#      http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+# License for the specific language governing permissions and limitations
11
+# under the License.
12
+
13
+from gyan.api import hooks
14
+
15
+# Pecan Application Configurations
16
+app = {
17
+    'root': 'gyan.api.controllers.root.RootController',
18
+    'modules': ['gyan'],
19
+    'hooks': [
20
+        hooks.ContextHook(),
21
+        hooks.NoExceptionTracebackHook(),
22
+        hooks.RPCHook(),
23
+    ],
24
+    'debug': True,
25
+}

+ 0
- 0
gyan/api/controllers/__init__.py View File


+ 233
- 0
gyan/api/controllers/base.py View File

@@ -0,0 +1,233 @@
1
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+#    not use this file except in compliance with the License. You may obtain
3
+#    a copy of the License at
4
+#
5
+#         http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+#    Unless required by applicable law or agreed to in writing, software
8
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+#    License for the specific language governing permissions and limitations
11
+#    under the License.
12
+
13
+import operator
14
+import six
15
+
16
+import pecan
17
+from pecan import rest
18
+from webob import exc
19
+from gyan.api.controllers import versions
20
+from gyan.api import versioned_method
21
+from gyan.common import exception
22
+from gyan.common.i18n import _
23
+
24
+
25
+# name of attribute to keep version method information
26
+VER_METHOD_ATTR = 'versioned_methods'
27
+
28
+
29
+class APIBase(object):
30
+
31
+    def __init__(self, **kwargs):
32
+        for field in self.fields:
33
+            if field in kwargs:
34
+                value = kwargs[field]
35
+                setattr(self, field, value)
36
+
37
+    def __setattr__(self, field, value):
38
+        super(APIBase, self).__setattr__(field, value)
39
+
40
+    def as_dict(self):
41
+        """Render this object as a dict of its fields."""
42
+        return {f: getattr(self, f)
43
+                for f in self.fields
44
+                if hasattr(self, f)}
45
+
46
+    def __json__(self):
47
+        return self.as_dict()
48
+
49
+    def unset_fields_except(self, except_list=None):
50
+        """Unset fields so they don't appear in the message body.
51
+
52
+        :param except_list: A list of fields that won't be touched.
53
+
54
+        """
55
+        if except_list is None:
56
+            except_list = []
57
+
58
+        for k in self.as_dict():
59
+            if k not in except_list:
60
+                setattr(self, k, None)
61
+
62
+
63
+class ControllerMetaclass(type):
64
+    """Controller metaclass.
65
+
66
+    This metaclass automates the task of assembling a dictionary
67
+    mapping action keys to method names.
68
+    """
69
+
70
+    def __new__(mcs, name, bases, cls_dict):
71
+        """Adds version function dictionary to the class."""
72
+
73
+        versioned_methods = None
74
+
75
+        for base in bases:
76
+            if base.__name__ == "Controller":
77
+                # NOTE(cyeoh): This resets the VER_METHOD_ATTR attribute
78
+                # between API controller class creations. This allows us
79
+                # to use a class decorator on the API methods that doesn't
80
+                # require naming explicitly what method is being versioned as
81
+                # it can be implicit based on the method decorated. It is a bit
82
+                # ugly.
83
+                if VER_METHOD_ATTR in base.__dict__:
84
+                    versioned_methods = getattr(base, VER_METHOD_ATTR)
85
+                    delattr(base, VER_METHOD_ATTR)
86
+
87
+        if versioned_methods:
88
+            cls_dict[VER_METHOD_ATTR] = versioned_methods
89
+
90
+        return super(ControllerMetaclass, mcs).__new__(mcs, name, bases,
91
+                                                       cls_dict)
92
+
93
+
94
+@six.add_metaclass(ControllerMetaclass)
95
+class Controller(rest.RestController):
96
+    """Base Rest Controller"""
97
+
98
+    @pecan.expose('json')
99
+    def _no_version_match(self, *args, **kwargs):
100
+        from pecan import request
101
+
102
+        raise exc.HTTPNotAcceptable(_(
103
+            "Version %(ver)s was requested but the requested API is not "
104
+            "supported for this version.") % {'ver': request.version})
105
+
106
+    def __getattribute__(self, key):
107
+
108
+        def version_select():
109
+            """Select the correct method based on version
110
+
111
+            @return: Returns the correct versioned method
112
+            @raises: HTTPNotAcceptable if there is no method which
113
+                 matches the name and version constraints
114
+            """
115
+
116
+            from pecan import request
117
+            ver = request.version
118
+
119
+            func_list = self.versioned_methods[key]
120
+            for func in func_list:
121
+                if ver.matches(func.start_version, func.end_version):
122
+                    return func.func
123
+
124
+            return self._no_version_match
125
+
126
+        try:
127
+            version_meth_dict = object.__getattribute__(self, VER_METHOD_ATTR)
128
+        except AttributeError:
129
+            # No versioning on this class
130
+            return object.__getattribute__(self, key)
131
+        if version_meth_dict and key in version_meth_dict:
132
+            return version_select().__get__(self, self.__class__)
133
+
134
+        return object.__getattribute__(self, key)
135
+
136
+    # NOTE: This decorator MUST appear first (the outermost
137
+    # decorator) on an API method for it to work correctly
138
+    @classmethod
139
+    def api_version(cls, min_ver, max_ver=None):
140
+        """Decorator for versioning api methods.
141
+
142
+        Add the decorator to any pecan method that has been exposed.
143
+        This decorator will store the method, min version, and max
144
+        version in a list for each api. It will check that there is no
145
+        overlap between versions and methods. When the api is called the
146
+        controller will use the list for each api to determine which
147
+        method to call.
148
+
149
+        Example:
150
+            @base.Controller.api_version("1.1", "1.2")
151
+            def get_one(self, ml_model_id):
152
+            {...code for versions 1.1 to 1.2...}
153
+
154
+            @base.Controller.api_version("1.3")
155
+            def get_one(self, ml_model_id):
156
+            {...code for versions 1.3 to latest}
157
+
158
+        @min_ver: string representing minimum version
159
+        @max_ver: optional string representing maximum version
160
+        @raises: ApiVersionsIntersect if an version overlap is found between
161
+        method versions.
162
+        """
163
+
164
+        def decorator(f):
165
+            obj_min_ver = versions.Version('', '', '', min_ver)
166
+            if max_ver:
167
+                obj_max_ver = versions.Version('', '', '', max_ver)
168
+            else:
169
+                obj_max_ver = versions.Version('', '', '',
170
+                                               versions.CURRENT_MAX_VER)
171
+
172
+            # Add to list of versioned methods registered
173
+            func_name = f.__name__
174
+            new_func = versioned_method.VersionedMethod(
175
+                func_name, obj_min_ver, obj_max_ver, f)
176
+
177
+            func_dict = getattr(cls, VER_METHOD_ATTR, {})
178
+            if not func_dict:
179
+                setattr(cls, VER_METHOD_ATTR, func_dict)
180
+
181
+            func_list = func_dict.get(func_name, [])
182
+            if not func_list:
183
+                func_dict[func_name] = func_list
184
+            func_list.append(new_func)
185
+
186
+            is_intersect = Controller.check_for_versions_intersection(
187
+                func_list)
188
+
189
+            if is_intersect:
190
+                raise exception.ApiVersionsIntersect(
191
+                    name=new_func.name,
192
+                    min_ver=new_func.start_version,
193
+                    max_ver=new_func.end_version
194
+                )
195
+
196
+            # Ensure the list is sorted by minimum version (reversed)
197
+            # so later when we work through the list in order we find
198
+            # the method which has the latest version which supports
199
+            # the version requested.
200
+            func_list.sort(key=lambda f: f.start_version, reverse=True)
201
+
202
+            return f
203
+
204
+        return decorator
205
+
206
+    @staticmethod
207
+    def check_for_versions_intersection(func_list):
208
+        """Determines whether function list intersections
209
+
210
+        General algorithm:
211
+        https://en.wikipedia.org/wiki/Intersection_algorithm
212
+
213
+        :param func_list: list of VersionedMethod objects
214
+        :return: boolean
215
+        """
216
+
217
+        pairs = []
218
+        counter = 0
219
+
220
+        for f in func_list:
221
+            pairs.append((f.start_version, 1))
222
+            pairs.append((f.end_version, -1))
223
+
224
+        pairs.sort(key=operator.itemgetter(1), reverse=True)
225
+        pairs.sort(key=operator.itemgetter(0))
226
+
227
+        for p in pairs:
228
+            counter += p[1]
229
+
230
+            if counter > 1:
231
+                return True
232
+
233
+        return False

+ 35
- 0
gyan/api/controllers/link.py View File

@@ -0,0 +1,35 @@
1
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+#    not use this file except in compliance with the License. You may obtain
3
+#    a copy of the License at
4
+#
5
+#         http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+#    Unless required by applicable law or agreed to in writing, software
8
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+#    License for the specific language governing permissions and limitations
11
+#    under the License.
12
+
13
+import pecan
14
+
15
+
16
+def build_url(resource, resource_args, bookmark=False, base_url=None):
17
+    if base_url is None:
18
+        base_url = pecan.request.host_url
19
+
20
+    template = '%(url)s/%(res)s' if bookmark else '%(url)s/v1/%(res)s'
21
+    # FIXME(tbh): I'm getting a 404 when doing a GET on
22
+    # a nested resource that the URL ends with a  '/'.
23
+    # https://groups.google.com/forum/#!topic/pecan-dev/QfSeviLg5qs
24
+    template += '%(args)s' if resource_args.startswith('?') else '/%(args)s'
25
+    return template % {'url': base_url, 'res': resource, 'args': resource_args}
26
+
27
+
28
+def make_link(rel_name, url, resource, resource_args,
29
+              bookmark=False, type=None):
30
+    href = build_url(resource, resource_args,
31
+                     bookmark=bookmark, base_url=url)
32
+    if type is None:
33
+        return {'href': href, 'rel': rel_name}
34
+    else:
35
+        return {'href': href, 'rel': rel_name, 'type': type}

+ 97
- 0
gyan/api/controllers/root.py View File

@@ -0,0 +1,97 @@
1
+#    Licensed under the Apache License, Version 2.0 (the "License");
2
+#    you may not use this file except in compliance with the License.
3
+#    You may obtain a copy of the License at
4
+#
5
+#        http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+#    Unless required by applicable law or agreed to in writing, software
8
+#    distributed under the License is distributed on an "AS IS" BASIS,
9
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10
+#    See the License for the specific language governing permissions and
11
+#    limitations under the License.
12
+
13
+import pecan
14
+from pecan import rest
15
+
16
+from gyan.api.controllers import base
17
+from gyan.api.controllers import link
18
+from gyan.api.controllers import v1
19
+from gyan.api.controllers import versions
20
+
21
+
22
+class Version(base.APIBase):
23
+    """An API version representation."""
24
+
25
+    fields = (
26
+        'id',
27
+        'links',
28
+        'status',
29
+        'max_version',
30
+        'min_version'
31
+    )
32
+
33
+    @staticmethod
34
+    def convert(id, status, max, min):
35
+        version = Version()
36
+        version.id = id
37
+        version.links = [link.make_link('self', pecan.request.host_url,
38
+                                        id, '', bookmark=True)]
39
+        version.status = status
40
+        version.max_version = max
41
+        version.min_version = min
42
+        return version
43
+
44
+
45
+class Root(base.APIBase):
46
+
47
+    fields = (
48
+        'name',
49
+        'description',
50
+        'versions',
51
+        'default_version',
52
+    )
53
+
54
+    @staticmethod
55
+    def convert():
56
+        root = Root()
57
+        root.name = "OpenStack Gyan API"
58
+        root.description = ("Gyan is an OpenStack project which aims to "
59
+                            "provide ML infra service.")
60
+
61
+        root.versions = [Version.convert('v1', "CURRENT",
62
+                                         versions.CURRENT_MAX_VER,
63
+                                         versions.BASE_VER)]
64
+        root.default_version = Version.convert('v1', "CURRENT",
65
+                                               versions.CURRENT_MAX_VER,
66
+                                               versions.BASE_VER)
67
+        return root
68
+
69
+
70
+class RootController(rest.RestController):
71
+
72
+    _versions = ['v1']
73
+    """All supported API versions"""
74
+
75
+    _default_version = 'v1'
76
+    """The default API version"""
77
+
78
+    v1 = v1.Controller()
79
+
80
+    @pecan.expose('json')
81
+    def get(self):
82
+        # NOTE: The reason why convert() it's being called for every
83
+        #       request is because we need to get the host url from
84
+        #       the request object to make the links.
85
+        return Root.convert()
86
+
87
+    @pecan.expose()
88
+    def _route(self, args):
89
+        """Overrides the default routing behavior.
90
+
91
+        It redirects the request to the default version of the gyan API
92
+        if the version number is not specified in the url.
93
+        """
94
+
95
+        if args[0] and args[0] not in self._versions:
96
+            args = [self._default_version] + args
97
+        return super(RootController, self)._route(args)

+ 155
- 0
gyan/api/controllers/v1/__init__.py View File

@@ -0,0 +1,155 @@
1
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+#    not use this file except in compliance with the License. You may obtain
3
+#    a copy of the License at
4
+#
5
+#         http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+#    Unless required by applicable law or agreed to in writing, software
8
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+#    License for the specific language governing permissions and limitations
11
+#    under the License.
12
+
13
+"""
14
+Version 1 of the Gyan API
15
+
16
+NOTE: IN PROGRESS AND NOT FULLY IMPLEMENTED.
17
+"""
18
+
19
+from oslo_log import log as logging
20
+import pecan
21
+
22
+from gyan.api.controllers import base as controllers_base
23
+from gyan.api.controllers import link
24
+from gyan.api.controllers.v1 import hosts as host_controller
25
+from gyan.api.controllers.v1 import ml_models as ml_model_controller
26
+from gyan.api.controllers import versions as ver
27
+from gyan.api import http_error
28
+from gyan.common.i18n import _
29
+
30
+LOG = logging.getLogger(__name__)
31
+
32
+
33
+BASE_VERSION = 1
34
+
35
+MIN_VER_STR = '%s %s' % (ver.Version.service_string, ver.BASE_VER)
36
+
37
+MAX_VER_STR = '%s %s' % (ver.Version.service_string, ver.CURRENT_MAX_VER)
38
+
39
+MIN_VER = ver.Version({ver.Version.string: MIN_VER_STR},
40
+                      MIN_VER_STR, MAX_VER_STR)
41
+MAX_VER = ver.Version({ver.Version.string: MAX_VER_STR},
42
+                      MIN_VER_STR, MAX_VER_STR)
43
+
44
+
45
+class MediaType(controllers_base.APIBase):
46
+    """A media type representation."""
47
+
48
+    fields = (
49
+        'base',
50
+        'type',
51
+    )
52
+
53
+
54
+class V1(controllers_base.APIBase):
55
+    """The representation of the version 1 of the API."""
56
+
57
+    fields = (
58
+        'id',
59
+        'media_types',
60
+        'links',
61
+        'hosts',
62
+        'ml_models'
63
+    )
64
+
65
+    @staticmethod
66
+    def convert():
67
+        v1 = V1()
68
+        v1.id = "v1"
69
+        v1.links = [link.make_link('self', pecan.request.host_url,
70
+                                   'v1', '', bookmark=True),
71
+                    link.make_link('describedby',
72
+                                   'https://docs.openstack.org',
73
+                                   'developer/gyan/dev',
74
+                                   'api-spec-v1.html',
75
+                                   bookmark=True, type='text/html')]
76
+        v1.media_types = [MediaType(base='application/json',
77
+                          type='application/vnd.openstack.gyan.v1+json')]
78
+        v1.hosts = [link.make_link('self', pecan.request.host_url,
79
+                                        'hosts', ''),
80
+                         link.make_link('bookmark',
81
+                                        pecan.request.host_url,
82
+                                        'hosts', '',
83
+                                        bookmark=True)]
84
+        v1.ml_models = [link.make_link('self', pecan.request.host_url,
85
+                                    'ml_models', ''),
86
+                     link.make_link('bookmark',
87
+                                    pecan.request.host_url,
88
+                                    'ml_models', '',
89
+                                    bookmark=True)]
90
+        return v1
91
+
92
+
93
+class Controller(controllers_base.Controller):
94
+    """Version 1 API controller root."""
95
+
96
+    hosts = host_controller.HostController()
97
+    ml_models = ml_model_controller.MLModelController()
98
+
99
+    @pecan.expose('json')
100
+    def get(self):
101
+        return V1.convert()
102
+
103
+    def _check_version(self, version, headers=None):
104
+        if headers is None:
105
+            headers = {}
106
+        # ensure that major version in the URL matches the header
107
+        if version.major != BASE_VERSION:
108
+            raise http_error.HTTPNotAcceptableAPIVersion(_(
109
+                "Mutually exclusive versions requested. Version %(ver)s "
110
+                "requested but not supported by this service. "
111
+                "The supported version range is: "
112
+                "[%(min)s, %(max)s].") % {'ver': version,
113
+                                          'min': MIN_VER_STR,
114
+                                          'max': MAX_VER_STR},
115
+                headers=headers,
116
+                max_version=str(MAX_VER),
117
+                min_version=str(MIN_VER))
118
+        # ensure the minor version is within the supported range
119
+        if version < MIN_VER or version > MAX_VER:
120
+            raise http_error.HTTPNotAcceptableAPIVersion(_(
121
+                "Version %(ver)s was requested but the minor version is not "
122
+                "supported by this service. The supported version range is: "
123
+                "[%(min)s, %(max)s].") % {'ver': version, 'min': MIN_VER_STR,
124
+                                          'max': MAX_VER_STR},
125
+                headers=headers,
126
+                max_version=str(MAX_VER),
127
+                min_version=str(MIN_VER))
128
+
129
+    @pecan.expose()
130
+    def _route(self, args):
131
+        version = ver.Version(
132
+            pecan.request.headers, MIN_VER_STR, MAX_VER_STR)
133
+
134
+        # Always set the basic version headers
135
+        pecan.response.headers[ver.Version.min_string] = MIN_VER_STR
136
+        pecan.response.headers[ver.Version.max_string] = MAX_VER_STR
137
+        pecan.response.headers[ver.Version.string] = " ".join(
138
+            [ver.Version.service_string, str(version)])
139
+        pecan.response.headers["vary"] = ver.Version.string
140
+
141
+        # assert that requested version is supported
142
+        self._check_version(version, pecan.response.headers)
143
+        pecan.request.version = version
144
+        if pecan.request.body:
145
+            msg = ("Processing request: url: %(url)s, %(method)s, "
146
+                   "body: %(body)s" %
147
+                   {'url': pecan.request.url,
148
+                    'method': pecan.request.method,
149
+                    'body': pecan.request.body})
150
+            LOG.debug(msg)
151
+
152
+        return super(Controller, self)._route(args)
153
+
154
+
155
+__all__ = ('Controller',)

+ 41
- 0
gyan/api/controllers/v1/collection.py View File

@@ -0,0 +1,41 @@
1
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+#    not use this file except in compliance with the License. You may obtain
3
+#    a copy of the License at
4
+#
5
+#         http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+#    Unless required by applicable law or agreed to in writing, software
8
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+#    License for the specific language governing permissions and limitations
11
+#    under the License.
12
+
13
+import pecan
14
+
15
+from gyan.api.controllers import base
16
+from gyan.api.controllers import link
17
+
18
+
19
+class Collection(base.APIBase):
20
+
21
+    @property
22
+    def collection(self):
23
+        return getattr(self, self._type)
24
+
25
+    def has_next(self, limit):
26
+        """Return whether collection has more items."""
27
+        return len(self.collection) and len(self.collection) == limit
28
+
29
+    def get_next(self, limit, url=None, **kwargs):
30
+        """Return a link to the next subset of the collection."""
31
+        if not self.has_next(limit):
32
+            return None
33
+
34
+        resource_url = url or self._type
35
+        q_args = ''.join(['%s=%s&' % (key, kwargs[key]) for key in kwargs])
36
+        next_args = '?%(args)slimit=%(limit)d&marker=%(marker)s' % {
37
+            'args': q_args, 'limit': limit,
38
+            'marker': self.collection[-1]['uuid']}
39
+
40
+        return link.make_link('next', pecan.request.host_url,
41
+                              resource_url, next_args)['href']

+ 109
- 0
gyan/api/controllers/v1/hosts.py View File

@@ -0,0 +1,109 @@
1
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+#    not use this file except in compliance with the License. You may obtain
3
+#    a copy of the License at
4
+#
5
+#         http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+#    Unless required by applicable law or agreed to in writing, software
8
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+#    License for the specific language governing permissions and limitations
11
+#    under the License.
12
+
13
+import pecan
14
+
15
+from gyan.api.controllers import base
16
+from gyan.api.controllers.v1 import collection
17
+from gyan.api.controllers.v1.views import hosts_view as view
18
+from gyan.api import utils as api_utils
19
+from gyan.common import exception
20
+from gyan.common import policy
21
+from gyan import objects
22
+
23
+
24
+def _get_host(host_ident):
25
+    host = api_utils.get_resource('ComputeHost', host_ident)
26
+    if not host:
27
+        pecan.abort(404, ('Not found; the host you requested '
28
+                          'does not exist.'))
29
+
30
+    return host
31
+
32
+
33
+def check_policy_on_host(host, action):
34
+    context = pecan.request.context
35
+    policy.enforce(context, action, host, action=action)
36
+
37
+
38
+class HostCollection(collection.Collection):
39
+    """API representation of a collection of hosts."""
40
+
41
+    fields = {
42
+        'hosts',
43
+        'next'
44
+    }
45
+
46
+    """A list containing compute host objects"""
47
+
48
+    def __init__(self, **kwargs):
49
+        super(HostCollection, self).__init__(**kwargs)
50
+        self._type = 'hosts'
51
+
52
+    @staticmethod
53
+    def convert_with_links(hosts, limit, url=None,
54
+                           expand=False, **kwargs):
55
+        collection = HostCollection()
56
+        collection.hosts = [view.format_host(url, p) for p in hosts]
57
+        collection.next = collection.get_next(limit, url=url, **kwargs)
58
+        return collection
59
+
60
+
61
+class HostController(base.Controller):
62
+    """Host info controller"""
63
+
64
+    @pecan.expose('json')
65
+    @base.Controller.api_version("1.0")
66
+    @exception.wrap_pecan_controller_exception
67
+    def get_all(self, **kwargs):
68
+        """Retrieve a list of hosts"""
69
+        context = pecan.request.context
70
+        policy.enforce(context, "host:get_all",
71
+                       action="host:get_all")
72
+        return self._get_host_collection(**kwargs)
73
+
74
+    def _get_host_collection(self, **kwargs):
75
+        context = pecan.request.context
76
+        limit = api_utils.validate_limit(kwargs.get('limit'))
77
+        sort_dir = api_utils.validate_sort_dir(kwargs.get('sort_dir', 'asc'))
78
+        sort_key = kwargs.get('sort_key', 'hostname')
79
+        expand = kwargs.get('expand')
80
+        filters = None
81
+        marker_obj = None
82
+        resource_url = kwargs.get('resource_url')
83
+        marker = kwargs.get('marker')
84
+        if marker:
85
+            marker_obj = objects.ComputeHost.get_by_uuid(context, marker)
86
+        hosts = objects.ComputeHost.list(context,
87
+                                         limit,
88
+                                         marker_obj,
89
+                                         sort_key,
90
+                                         sort_dir,
91
+                                         filters=filters)
92
+        return HostCollection.convert_with_links(hosts, limit,
93
+                                                 url=resource_url,
94
+                                                 expand=expand,
95
+                                                 sort_key=sort_key,
96
+                                                 sort_dir=sort_dir)
97
+
98
+    @pecan.expose('json')
99
+    @base.Controller.api_version("1.0")
100
+    @exception.wrap_pecan_controller_exception
101
+    def get_one(self, host_ident):
102
+        """Retrieve information about the given host.
103
+
104
+        :param host_ident: UUID or name of a host.
105
+        """
106
+        context = pecan.request.context
107
+        policy.enforce(context, "host:get", action="host:get")
108
+        host = _get_host(host_ident)
109
+        return view.format_host(pecan.request.host_url, host)

+ 289
- 0
gyan/api/controllers/v1/ml_models.py View File

@@ -0,0 +1,289 @@
1
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+#    not use this file except in compliance with the License. You may obtain
3
+#    a copy of the License at
4
+#
5
+#         http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+#    Unless required by applicable law or agreed to in writing, software
8
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+#    License for the specific language governing permissions and limitations
11
+#    under the License.
12
+
13
+import shlex
14
+
15
+from oslo_log import log as logging
16
+from oslo_utils import strutils
17
+from oslo_utils import uuidutils
18
+import pecan
19
+import six
20
+
21
+from gyan.api.controllers import base
22
+from gyan.api.controllers import link
23
+from gyan.api.controllers.v1 import collection
24
+from gyan.api.controllers.v1.schemas import ml_models as schema
25
+from gyan.api.controllers.v1.views import ml_models_view as view
26
+from gyan.api import utils as api_utils
27
+from gyan.api import validation
28
+from gyan.common import consts
29
+from gyan.common import context as gyan_context
30
+from gyan.common import exception
31
+from gyan.common.i18n import _
32
+from gyan.common.policies import ml_model as policies
33
+from gyan.common import policy
34
+from gyan.common import utils
35
+import gyan.conf
36
+from gyan import objects
37
+
38
+CONF = gyan.conf.CONF
39
+LOG = logging.getLogger(__name__)
40
+
41
+
42
+def check_policy_on_ml_model(ml_model, action):
43
+    context = pecan.request.context
44
+    policy.enforce(context, action, ml_model, action=action)
45
+
46
+
47
+class MLModelCollection(collection.Collection):
48
+    """API representation of a collection of ml models."""
49
+
50
+    fields = {
51
+        'ml_models',
52
+        'next'
53
+    }
54
+
55
+    """A list containing ml models objects"""
56
+
57
+    def __init__(self, **kwargs):
58
+        super(MLModelCollection, self).__init__(**kwargs)
59
+        self._type = 'ml_models'
60
+
61
+    @staticmethod
62
+    def convert_with_links(rpc_ml_models, limit, url=None,
63
+                           expand=False, **kwargs):
64
+        context = pecan.request.context
65
+        collection = MLModelCollection()
66
+        collection.ml_models = \
67
+            [view.format_ml_model(context, url, p.as_dict())
68
+             for p in rpc_ml_models]
69
+        collection.next = collection.get_next(limit, url=url, **kwargs)
70
+        return collection
71
+
72
+
73
+class MLModelController(base.Controller):
74
+    """Controller for MLModels."""
75
+
76
+    _custom_actions = {
77
+        'train': ['POST'],
78
+        'deploy': ['GET'],
79
+        'undeploy': ['GET']
80
+    }
81
+
82
+    
83
+    @pecan.expose('json')
84
+    @exception.wrap_pecan_controller_exception
85
+    def get_all(self, **kwargs):
86
+        """Retrieve a list of ml models.
87
+
88
+        """
89
+        context = pecan.request.context
90
+        policy.enforce(context, "ml_model:get_all",
91
+                       action="ml_model:get_all")
92
+        return self._get_ml_models_collection(**kwargs)
93
+
94
+    def _get_ml_models_collection(self, **kwargs):
95
+        context = pecan.request.context
96
+        if utils.is_all_projects(kwargs):
97
+            policy.enforce(context, "ml_model:get_all_all_projects",
98
+                           action="ml_model:get_all_all_projects")
99
+            context.all_projects = True
100
+        kwargs.pop('all_projects', None)
101
+        limit = api_utils.validate_limit(kwargs.pop('limit', None))
102
+        sort_dir = api_utils.validate_sort_dir(kwargs.pop('sort_dir', 'asc'))
103
+        sort_key = kwargs.pop('sort_key', 'id')
104
+        resource_url = kwargs.pop('resource_url', None)
105
+        expand = kwargs.pop('expand', None)
106
+
107
+        ml_model_allowed_filters = ['name', 'status', 'project_id', 'user_id',
108
+                                     'type']
109
+        filters = {}
110
+        for filter_key in ml_model_allowed_filters:
111
+            if filter_key in kwargs:
112
+                policy_action = policies.MLMODEL % ('get_one:' + filter_key)
113
+                context.can(policy_action, might_not_exist=True)
114
+                filter_value = kwargs.pop(filter_key)
115
+                filters[filter_key] = filter_value
116
+        marker_obj = None
117
+        marker = kwargs.pop('marker', None)
118
+        if marker:
119
+            marker_obj = objects.ML_Model.get_by_uuid(context,
120
+                                                       marker)
121
+        if kwargs:
122
+            unknown_params = [str(k) for k in kwargs]
123
+            msg = _("Unknown parameters: %s") % ", ".join(unknown_params)
124
+            raise exception.InvalidValue(msg)
125
+
126
+        ml_models = objects.ML_Model.list(context,
127
+                                            limit,
128
+                                            marker_obj,
129
+                                            sort_key,
130
+                                            sort_dir,
131
+                                            filters=filters)
132
+        return MLModelCollection.convert_with_links(ml_models, limit,
133
+                                                      url=resource_url,
134
+                                                      expand=expand,
135
+                                                      sort_key=sort_key,
136
+                                                      sort_dir=sort_dir)
137
+
138
+    @pecan.expose('json')
139
+    @exception.wrap_pecan_controller_exception
140
+    def get_one(self, ml_model_ident, **kwargs):
141
+        """Retrieve information about the given ml_model.
142
+
143
+        :param ml_model_ident: UUID or name of a ml_model.
144
+        """
145
+        context = pecan.request.context
146
+        if utils.is_all_projects(kwargs):
147
+            policy.enforce(context, "ml_model:get_one_all_projects",
148
+                           action="ml_model:get_one_all_projects")
149
+            context.all_projects = True
150
+        ml_model = utils.get_ml_model(ml_model_ident)
151
+        check_policy_on_ml_model(ml_model.as_dict(), "ml_model:get_one")
152
+        if ml_model.node:
153
+            compute_api = pecan.request.compute_api
154
+            try:
155
+                ml_model = compute_api.ml_model_show(context, ml_model)
156
+            except exception.MLModelHostNotUp:
157
+                raise exception.ServerNotUsable
158
+
159
+        return view.format_ml_model(context, pecan.request.host_url,
160
+                                     ml_model.as_dict())
161
+
162
+    @base.Controller.api_version("1.0")
163
+    @pecan.expose('json')
164
+    @api_utils.enforce_content_types(['application/json'])
165
+    @exception.wrap_pecan_controller_exception
166
+    @validation.validate_query_param(pecan.request, schema.query_param_create)
167
+    @validation.validated(schema.ml_model_create)
168
+    def post(self, **ml_model_dict):
169
+        return self._do_post(**ml_model_dict)
170
+
171
+
172
+    def _do_post(self, **ml_model_dict):
173
+        """Create or run a new ml model.
174
+
175
+        :param ml_model_dict: a ml_model within the request body.
176
+        """
177
+        context = pecan.request.context
178
+        compute_api = pecan.request.compute_api
179
+        policy.enforce(context, "ml_model:create",
180
+                       action="ml_model:create")
181
+
182
+        ml_model_dict['project_id'] = context.project_id
183
+        ml_model_dict['user_id'] = context.user_id
184
+        name = ml_model_dict.get('name')
185
+        ml_model_dict['name'] = name
186
+        
187
+        ml_model_dict['status'] = consts.CREATING
188
+        extra_spec = {}
189
+        extra_spec['hints'] = ml_model_dict.get('hints', None)
190
+        new_ml_model = objects.ML_Model(context, **ml_model_dict)
191
+        new_ml_model.create(context)
192
+
193
+        compute_api.ml_model_create(context, new_ml_model, **kwargs)
194
+        # Set the HTTP Location Header
195
+        pecan.response.location = link.build_url('ml_models',
196
+                                                 new_ml_model.uuid)
197
+        pecan.response.status = 202
198
+        return view.format_ml_model(context, pecan.request.node_url,
199
+                                     new_ml_model.as_dict())
200
+
201
+    
202
+    @pecan.expose('json')
203
+    @exception.wrap_pecan_controller_exception
204
+    @validation.validated(schema.ml_model_update)
205
+    def patch(self, ml_model_ident, **patch):
206
+        """Update an existing ml model.
207
+
208
+        :param ml_model_ident: UUID or name of a ml model.
209
+        :param patch: a json PATCH document to apply to this ml model.
210
+        """
211
+        ml_model = utils.get_ml_model(ml_model_ident)
212
+        check_policy_on_ml_model(ml_model.as_dict(), "ml_model:update")
213
+        utils.validate_ml_model_state(ml_model, 'update')
214
+        context = pecan.request.context
215
+        compute_api = pecan.request.compute_api
216
+        ml_model = compute_api.ml_model_update(context, ml_model, patch)
217
+        return view.format_ml_model(context, pecan.request.node_url,
218
+                                     ml_model.as_dict())
219
+
220
+    
221
+    @pecan.expose('json')
222
+    @exception.wrap_pecan_controller_exception
223
+    @validation.validate_query_param(pecan.request, schema.query_param_delete)
224
+    def delete(self, ml_model_ident, force=False, **kwargs):
225
+        """Delete a ML Model.
226
+
227
+        :param ml_model_ident: UUID or Name of a ML Model.
228
+        :param force: If True, allow to force delete the ML Model.
229
+        """
230
+        context = pecan.request.context
231
+        ml_model = utils.get_ml_model(ml_model_ident)
232
+        check_policy_on_ml_model(ml_model.as_dict(), "ml_model:delete")
233
+        try:
234
+            force = strutils.bool_from_string(force, strict=True)
235
+        except ValueError:
236
+            bools = ', '.join(strutils.TRUE_STRINGS + strutils.FALSE_STRINGS)
237
+            raise exception.InvalidValue(_('Valid force values are: %s')
238
+                                         % bools)
239
+        stop = kwargs.pop('stop', False)
240
+        try:
241
+            stop = strutils.bool_from_string(stop, strict=True)
242
+        except ValueError:
243
+            bools = ', '.join(strutils.TRUE_STRINGS + strutils.FALSE_STRINGS)
244
+            raise exception.InvalidValue(_('Valid stop values are: %s')
245
+                                         % bools)
246
+        compute_api = pecan.request.compute_api
247
+        if not force:
248
+            utils.validate_ml_model_state(ml_model, 'delete')
249
+        ml_model.status = consts.DELETING
250
+        if ml_model.node:
251
+            compute_api.ml_model_delete(context, ml_model, force)
252
+        else:
253
+            ml_model.destroy(context)
254
+        pecan.response.status = 204
255
+
256
+
257
+    @pecan.expose('json')
258
+    @exception.wrap_pecan_controller_exception
259
+    def deploy(self, ml_model_ident, **kwargs):
260
+        """Deploy ML Model.
261
+
262
+        :param ml_model_ident: UUID or Name of a ML Model.
263
+        """
264
+        ml_model = utils.get_ml_model(ml_model_ident)
265
+        check_policy_on_ml_model(ml_model.as_dict(), "ml_model:deploy")
266
+        utils.validate_ml_model_state(ml_model, 'deploy')
267
+        LOG.debug('Calling compute.ml_model_deploy with %s',
268
+                  ml_model.uuid)
269
+        context = pecan.request.context
270
+        compute_api = pecan.request.compute_api
271
+        compute_api.ml_model_deploy(context, ml_model)
272
+        pecan.response.status = 202
273
+
274
+    @pecan.expose('json')
275
+    @exception.wrap_pecan_controller_exception
276
+    def undeploy(self, ml_model_ident, **kwargs):
277
+        """Undeploy ML Model.
278
+
279
+        :param ml_model_ident: UUID or Name of a ML Model.
280
+        """
281
+        ml_model = utils.get_ml_model(ml_model_ident)
282
+        check_policy_on_ml_model(ml_model.as_dict(), "ml_model:deploy")
283
+        utils.validate_ml_model_state(ml_model, 'undeploy')
284
+        LOG.debug('Calling compute.ml_model_deploy with %s',
285
+                  ml_model.uuid)
286
+        context = pecan.request.context
287
+        compute_api = pecan.request.compute_api
288
+        compute_api.ml_model_undeploy(context, ml_model)
289
+        pecan.response.status = 202

+ 0
- 0
gyan/api/controllers/v1/schemas/__init__.py View File


+ 49
- 0
gyan/api/controllers/v1/schemas/ml_models.py View File

@@ -0,0 +1,49 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+# not use this file except in compliance with the License. You may obtain
3
+# a copy of the License at
4
+#
5
+# http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+# License for the specific language governing permissions and limitations
11
+# under the License.
12
+
13
+import copy
14
+
15
+from gyan.api.controllers.v1.schemas import parameter_types
16
+
17
+_ml_model_properties = {}
18
+
19
+ml_model_create = {
20
+    'type': 'object',
21
+    'properties': _ml_model_properties,
22
+    'required': ['name'],
23
+    'additionalProperties': False
24
+}
25
+
26
+
27
+query_param_create = {
28
+    'type': 'object',
29
+    'properties': {
30
+        'run': parameter_types.boolean_extended
31
+    },
32
+    'additionalProperties': False
33
+}
34
+
35
+ml_model_update = {
36
+    'type': 'object',
37
+    'properties': {},
38
+    'additionalProperties': False
39
+}
40
+
41
+query_param_delete = {
42
+    'type': 'object',
43
+    'properties': {
44
+        'force': parameter_types.boolean_extended,
45
+        'all_projects': parameter_types.boolean_extended,
46
+        'stop': parameter_types.boolean_extended
47
+    },
48
+    'additionalProperties': False
49
+}

+ 97
- 0
gyan/api/controllers/v1/schemas/parameter_types.py View File

@@ -0,0 +1,97 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+# not use this file except in compliance with the License. You may obtain
3
+# a copy of the License at
4
+#
5
+# http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+# License for the specific language governing permissions and limitations
11
+# under the License.
12
+
13
+import copy
14
+import signal
15
+import sys
16
+import gyan.conf
17
+
18
+CONF = gyan.conf.CONF
19
+
20
+non_negative_integer = {
21
+    'type': ['integer', 'string'],
22
+    'pattern': '^[0-9]*$', 'minimum': 0
23
+}
24
+
25
+positive_integer = {
26
+    'type': ['integer', 'string'],
27
+    'pattern': '^[0-9]*$', 'minimum': 1
28
+}
29
+
30
+boolean_extended = {
31
+    'type': ['boolean', 'string'],
32
+    'enum': [True, 'True', 'TRUE', 'true', '1', 'ON', 'On', 'on',
33
+             'YES', 'Yes', 'yes',
34
+             False, 'False', 'FALSE', 'false', '0', 'OFF', 'Off', 'off',
35
+             'NO', 'No', 'no'],
36
+}
37
+
38
+boolean = {
39
+    'type': ['boolean', 'string'],
40
+    'enum': [True, 'True', 'true', False, 'False', 'false'],
41
+}
42
+
43
+ml_model_name = {
44
+    'type': ['string', 'null'],
45
+    'minLength': 2,
46
+    'maxLength': 255,
47
+    'pattern': '^[a-zA-Z0-9][a-zA-Z0-9_.-]+$'
48
+}
49
+
50
+hex_uuid = {
51
+    'type': 'string',
52
+    'maxLength': 32,
53
+    'minLength': 32,
54
+    'pattern': '^[a-fA-F0-9]*$'
55
+}
56
+
57
+
58
+labels = {
59
+    'type': ['object', 'null']
60
+}
61
+
62
+hints = {
63
+    'type': ['object', 'null']
64
+}
65
+hostname = {
66
+    'type': ['string', 'null'],
67
+    'minLength': 2,
68
+    'maxLength': 63
69
+}
70
+
71
+repo = {
72
+    'type': 'string',
73
+    'minLength': 2,
74
+    'maxLength': 255,
75
+    'pattern': '[a-zA-Z0-9][a-zA-Z0-9_.-]'
76
+}
77
+
78
+
79
+
80
+string_ps_args = {
81
+    'type': ['string'],
82
+    'pattern': '[a-zA-Z- ,+]*'
83
+}
84
+
85
+str_and_int = {
86
+    'type': ['string', 'integer', 'null'],
87
+}
88
+
89
+hostname = {
90
+    'type': 'string', 'minLength': 1, 'maxLength': 255,
91
+    # NOTE: 'host' is defined in "services" table, and that
92
+    # means a hostname. The hostname grammar in RFC952 does
93
+    # not allow for underscores in hostnames. However, this
94
+    # schema allows them, because it sometimes occurs in
95
+    # real systems.
96
+    'pattern': '^[a-zA-Z0-9-._]*$',
97
+}

+ 50
- 0
gyan/api/controllers/v1/schemas/services.py View File

@@ -0,0 +1,50 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+# not use this file except in compliance with the License. You may obtain
3
+# a copy of the License at
4
+#
5
+# http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+# License for the specific language governing permissions and limitations
11
+# under the License.
12
+
13
+from gyan.api.controllers.v1.schemas import parameter_types
14
+
15
+query_param_enable = {
16
+    'type': 'object',
17
+    'properties': {
18
+        'host': parameter_types.hostname,
19
+        'binary': {
20
+            'type': 'string', 'minLength': 1, 'maxLength': 255,
21
+        },
22
+    },
23
+    'additionalProperties': False
24
+}
25
+
26
+query_param_disable = {
27
+    'type': 'object',
28
+    'properties': {
29
+        'host': parameter_types.hostname,
30
+        'binary': {
31
+            'type': 'string', 'minLength': 1, 'maxLength': 255,
32
+        },
33
+        'disabled_reason': {
34
+            'type': 'string', 'minLength': 1, 'maxLength': 255,
35
+        },
36
+    },
37
+    'additionalProperties': False
38
+}
39
+
40
+query_param_force_down = {
41
+    'type': 'object',
42
+    'properties': {
43
+        'host': parameter_types.hostname,
44
+        'binary': {
45
+            'type': 'string', 'minLength': 1, 'maxLength': 255,
46
+        },
47
+        'forced_down': parameter_types.boolean
48
+    },
49
+    'additionalProperties': False
50
+}

+ 0
- 0
gyan/api/controllers/v1/views/__init__.py View File


+ 43
- 0
gyan/api/controllers/v1/views/hosts_view.py View File

@@ -0,0 +1,43 @@
1
+#
2
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
3
+#    not use this file except in compliance with the License. You may obtain
4
+#    a copy of the License at
5
+#
6
+#         http://www.apache.org/licenses/LICENSE-2.0
7
+#
8
+#    Unless required by applicable law or agreed to in writing, software
9
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
10
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
11
+#    License for the specific language governing permissions and limitations
12
+#    under the License.
13
+
14
+import itertools
15
+
16
+from gyan.api.controllers import link
17
+
18
+
19
+_basic_keys = (
20
+    'id',
21
+    'hostname',
22
+    'type',
23
+    'status'
24
+)
25
+
26
+
27
+def format_host(url, host):
28
+    def transform(key, value):
29
+        if key not in _basic_keys:
30
+            return
31
+        if key == 'id':
32
+            yield ('id', value)
33
+            yield ('links', [link.make_link(
34
+                'self', url, 'hosts', value),
35
+                link.make_link(
36
+                    'bookmark', url,
37
+                    'hosts', value,
38
+                    bookmark=True)])
39
+        else:
40
+            yield (key, value)
41
+
42
+    return dict(itertools.chain.from_iterable(
43
+        transform(k, v) for k, v in host.as_dict().items()))

+ 55
- 0
gyan/api/controllers/v1/views/ml_models_view.py View File

@@ -0,0 +1,55 @@
1
+#
2
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
3
+#    not use this file except in compliance with the License. You may obtain
4
+#    a copy of the License at
5
+#
6
+#         http://www.apache.org/licenses/LICENSE-2.0
7
+#
8
+#    Unless required by applicable law or agreed to in writing, software
9
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
10
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
11
+#    License for the specific language governing permissions and limitations
12
+#    under the License.
13
+
14
+import itertools
15
+
16
+from gyan.api.controllers import link
17
+from gyan.common.policies import ml_model as policies
18
+
19
+_basic_keys = (
20
+    'uuid',
21
+    'user_id',
22
+    'project_id',
23
+    'name',
24
+    'url',
25
+    'status',
26
+    'status_reason',
27
+    'task_state',
28
+    'labels',
29
+    'host',
30
+    'status_detail'
31
+)
32
+
33
+
34
+def format_ml_model(context, url, ml_model):
35
+    def transform(key, value):
36
+        if key not in _basic_keys:
37
+            return
38
+        # strip the key if it is not allowed by policy
39
+        policy_action = policies.ML_MODEL % ('get_one:%s' % key)
40
+        if not context.can(policy_action, fatal=False, might_not_exist=True):
41
+            return
42
+        if key == 'uuid':
43
+            yield ('uuid', value)
44
+            if url:
45
+                yield ('links', [link.make_link(
46
+                    'self', url, 'ml_models', value),
47
+                    link.make_link(
48
+                        'bookmark', url,
49
+                        'ml_models', value,
50
+                        bookmark=True)])
51
+        else:
52
+            yield (key, value)
53
+
54
+    return dict(itertools.chain.from_iterable(
55
+        transform(k, v) for k, v in ml_model.items()))

+ 145
- 0
gyan/api/controllers/versions.py View File

@@ -0,0 +1,145 @@
1
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+#    not use this file except in compliance with the License. You may obtain
3
+#    a copy of the License at
4
+#
5
+#         http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+#    Unless required by applicable law or agreed to in writing, software
8
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+#    License for the specific language governing permissions and limitations
11
+#    under the License.
12
+
13
+
14
+from webob import exc
15
+
16
+from gyan.common.i18n import _
17
+
18
+# NOTE(tbh):  v1.0 is reserved to indicate Ocata's API, but is not presently
19
+#             supported by the API service. All changes between Ocata and the
20
+#             point where we added microversioning are considered backwards-
21
+#             compatible, but are not specifically discoverable at this time.
22
+#
23
+#             The v1.1 version indicates this "initial" version as being
24
+#             different from Ocata (v1.0), and includes the following changes:
25
+#
26
+#             Add details of new api versions here:
27
+
28
+#
29
+# For each newly added microversion change, update the API version history
30
+# string below with a one or two line description. Also update
31
+# rest_api_version_history.rst for extra information on microversion.
32
+REST_API_VERSION_HISTORY = """REST API Version History:
33
+
34
+    * 1.1 - Initial version
35
+"""
36
+
37
+BASE_VER = '1.1'
38
+CURRENT_MAX_VER = '1.1'
39
+
40
+
41
+class Version(object):
42
+    """API Version object."""
43
+
44
+    string = 'OpenStack-API-Version'
45
+    """HTTP Header string carrying the requested version"""
46
+
47
+    min_string = 'OpenStack-API-Minimum-Version'
48
+    """HTTP response header"""
49
+
50
+    max_string = 'OpenStack-API-Maximum-Version'
51
+    """HTTP response header"""
52
+
53
+    service_string = 'ml'
54
+
55
+    def __init__(self, headers, default_version, latest_version,
56
+                 from_string=None):
57
+        """Create an API Version object from the supplied headers.
58
+
59
+        :param headers: webob headers
60
+        :param default_version: version to use if not specified in headers
61
+        :param latest_version: version to use if latest is requested
62
+        :param from_string: create the version from string not headers
63
+        :raises: webob.HTTPNotAcceptable
64
+        """
65
+        if from_string:
66
+            (self.major, self.minor) = tuple(int(i)
67
+                                             for i in from_string.split('.'))
68
+
69
+        else:
70
+            (self.major, self.minor) = Version.parse_headers(headers,
71
+                                                             default_version,
72
+                                                             latest_version)
73
+
74
+    def __repr__(self):
75
+        return '%s.%s' % (self.major, self.minor)
76
+
77
+    @staticmethod
78
+    def parse_headers(headers, default_version, latest_version):
79
+        """Determine the API version requested based on the headers supplied.
80
+
81
+        :param headers: webob headers
82
+        :param default_version: version to use if not specified in headers
83
+        :param latest_version: version to use if latest is requested
84
+        :returns: a tuple of (major, minor) version numbers
85
+        :raises: webob.HTTPNotAcceptable
86
+        """
87
+
88
+        version_hdr = headers.get(Version.string, default_version)
89
+
90
+        try:
91
+            version_service, version_str = version_hdr.split()
92
+        except ValueError:
93
+            raise exc.HTTPNotAcceptable(_(
94
+                "Invalid service type for %s header") % Version.string)
95
+
96
+        if version_str.lower() == 'latest':
97
+            version_service, version_str = latest_version.split()
98
+
99
+        if version_service != Version.service_string:
100
+            raise exc.HTTPNotAcceptable(_(
101
+                "Invalid service type for %s header") % Version.string)
102
+        try:
103
+            version = tuple(int(i) for i in version_str.split('.'))
104
+        except ValueError:
105
+            version = ()
106
+
107
+        if len(version) != 2:
108
+            raise exc.HTTPNotAcceptable(_(
109
+                "Invalid value for %s header") % Version.string)
110
+        return version
111
+
112
+    def is_null(self):
113
+        return self.major == 0 and self.minor == 0
114
+
115
+    def matches(self, start_version, end_version):
116
+        if self.is_null():
117
+            raise ValueError
118
+
119
+        return start_version <= self <= end_version
120
+
121
+    def __lt__(self, other):
122
+        if self.major < other.major:
123
+            return True
124
+        if self.major == other.major and self.minor < other.minor:
125
+            return True
126
+        return False
127
+
128
+    def __gt__(self, other):
129
+        if self.major > other.major:
130
+            return True
131
+        if self.major == other.major and self.minor > other.minor:
132
+            return True
133
+        return False
134
+
135
+    def __eq__(self, other):
136
+        return self.major == other.major and self.minor == other.minor
137
+
138
+    def __le__(self, other):
139
+        return self < other or self == other
140
+
141
+    def __ne__(self, other):
142
+        return not self.__eq__(other)
143
+
144
+    def __ge__(self, other):
145
+        return self > other or self == other

+ 114
- 0
gyan/api/hooks.py View File

@@ -0,0 +1,114 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+# not use this file except in compliance with the License. You may obtain
3
+# a copy of the License at
4
+#
5
+#      http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+# License for the specific language governing permissions and limitations
11
+# under the License.
12
+
13
+
14
+from pecan import hooks
15
+
16
+from gyan.common import context
17
+from gyan.compute import api as compute_api
18
+import gyan.conf
19
+
20
+CONF = gyan.conf.CONF
21
+
22
+
23
+class ContextHook(hooks.PecanHook):
24
+    """Configures a request context and attaches it to the request.
25
+
26
+    The following HTTP request headers are used:
27
+
28
+    X-User-Name:
29
+        Used for context.user_name.
30
+
31
+    X-User-Id:
32
+        Used for context.user_id.
33
+
34
+    X-Project-Name:
35
+        Used for context.project.
36
+
37
+    X-Project-Id:
38
+        Used for context.project_id.
39
+
40
+    X-Auth-Token:
41
+        Used for context.auth_token.
42
+
43
+    X-Roles:
44
+        Used for context.roles.
45
+    """
46
+
47
+    def before(self, state):
48
+        headers = state.request.headers
49
+        user_name = headers.get('X-User-Name')
50
+        user_id = headers.get('X-User-Id')
51
+        project = headers.get('X-Project-Name')
52
+        project_id = headers.get('X-Project-Id')
53
+        domain_id = headers.get('X-User-Domain-Id')
54
+        domain_name = headers.get('X-User-Domain-Name')
55
+        auth_token = headers.get('X-Auth-Token')
56
+        roles = headers.get('X-Roles', '').split(',')
57
+        auth_token_info = state.request.environ.get('keystone.token_info')
58
+
59
+        state.request.context = context.make_context(
60
+            auth_token=auth_token,
61
+            auth_token_info=auth_token_info,
62
+            user_name=user_name,
63
+            user_id=user_id,
64
+            project_name=project,
65
+            project_id=project_id,
66
+            domain_id=domain_id,
67
+            domain_name=domain_name,
68
+            roles=roles)
69
+
70
+
71
+class RPCHook(hooks.PecanHook):
72
+    """Attach the rpcapi object to the request so controllers can get to it."""
73
+
74
+    def before(self, state):
75
+        context = state.request.context
76
+        state.request.compute_api = compute_api.API(context)
77
+
78
+
79
+class NoExceptionTracebackHook(hooks.PecanHook):
80
+    """Workaround rpc.common: deserialize_remote_exception.
81
+
82
+    deserialize_remote_exception builds rpc exception traceback into error
83
+    message which is then sent to the client. Such behavior is a security
84
+    concern so this hook is aimed to cut-off traceback from the error message.
85
+    """
86
+    # NOTE(tbh): 'after' hook used instead of 'on_error' because
87
+    # 'on_error' never fired for wsme+pecan pair. wsme @wsexpose decorator
88
+    # catches and handles all the errors, so 'on_error' dedicated for unhandled
89
+    # exceptions never fired.
90
+    def after(self, state):
91
+        # Omit empty body. Some errors may not have body at this level yet.
92
+        if not state.response.body:
93
+            return
94
+
95
+        # Do nothing if there is no error.
96
+        if 200 <= state.response.status_int < 400:
97
+            return
98
+
99
+        json_body = state.response.json
100
+        # Do not remove traceback when server in debug mode (except 'Server'
101
+        # errors when 'debuginfo' will be used for traces).
102
+        if CONF.debug and json_body.get('faultcode') != 'Server':
103
+            return
104
+
105
+        title = json_body.get('title')
106
+        traceback_marker = 'Traceback (most recent call last):'
107
+        if title and (traceback_marker in title):
108
+            # Cut-off traceback.
109
+            title = title.split(traceback_marker, 1)[0]
110
+            # Remove trailing newlines and spaces if any.
111
+            json_body['title'] = title.rstrip()
112
+            # Replace the whole json. Cannot change original one beacause it's
113
+            # generated on the fly.
114
+            state.response.json = json_body

+ 69
- 0
gyan/api/http_error.py View File

@@ -0,0 +1,69 @@
1
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+#    not use this file except in compliance with the License. You may obtain
3
+#    a copy of the License at
4
+#
5
+#         http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+#    Unless required by applicable law or agreed to in writing, software
8
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+#    License for the specific language governing permissions and limitations
11
+#    under the License.
12
+
13
+import six
14
+
15
+from oslo_serialization import jsonutils as json
16
+from webob import exc
17
+
18
+
19
+class HTTPNotAcceptableAPIVersion(exc.HTTPNotAcceptable):
20
+    # subclass of :class:`~HTTPNotAcceptable`
21
+    #
22
+    # This indicates the resource identified by the request is only
23
+    # capable of generating response entities which have content
24
+    # characteristics not acceptable according to the accept headers
25
+    # sent in the request.
26
+    #
27
+    # code: 406, title: Not Acceptable
28
+    #
29
+    # differences from webob.exc.HTTPNotAcceptable:
30
+    #
31
+    # - additional max and min version parameters
32
+    # - additional error info for code, title, and links
33
+    code = 406
34
+    title = 'Not Acceptable'
35
+    max_version = ''
36
+    min_version = ''
37
+
38
+    def __init__(self, detail=None, headers=None, comment=None,
39
+                 body_template=None, max_version='', min_version='', **kwargs):
40
+
41
+        super(HTTPNotAcceptableAPIVersion, self).__init__(
42
+            detail=detail, headers=headers, comment=comment,
43
+            body_template=body_template, **kwargs)
44
+
45
+        self.max_version = max_version
46
+        self.min_version = min_version
47
+
48
+    def __call__(self, environ, start_response):
49
+        for err_str in self.app_iter:
50
+            err = {}
51
+            try:
52
+                err = json.loads(err_str.decode('utf-8'))
53
+            except ValueError:
54
+                pass
55
+
56
+            links = {'rel': 'help', 'href': 'https://developer.openstack.org'
57
+                     '/api-guide/compute/microversions.html'}
58
+
59
+            err['max_version'] = self.max_version
60
+            err['min_version'] = self.min_version
61
+            err['code'] = "gyan.microversion-unsupported"
62
+            err['links'] = [links]
63
+            err['title'] = "Requested microversion is unsupported"
64
+
65
+        self.app_iter = [six.b(json.dump_as_bytes(err))]
66
+        self.headers['Content-Length'] = str(len(self.app_iter[0]))
67
+
68
+        return super(HTTPNotAcceptableAPIVersion, self).__call__(
69
+            environ, start_response)

+ 21
- 0
gyan/api/middleware/__init__.py View File

@@ -0,0 +1,21 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+# not use this file except in compliance with the License. You may obtain
3
+# a copy of the License at
4
+#
5
+#      http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+# License for the specific language governing permissions and limitations
11
+# under the License.
12
+
13
+from gyan.api.middleware import auth_token
14
+from gyan.api.middleware import parsable_error
15
+
16
+
17
+AuthTokenMiddleware = auth_token.AuthTokenMiddleware
18
+ParsableErrorMiddleware = parsable_error.ParsableErrorMiddleware
19
+
20
+__all__ = ('AuthTokenMiddleware',
21
+           'ParsableErrorMiddleware')

+ 71
- 0
gyan/api/middleware/auth_token.py View File

@@ -0,0 +1,71 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+# not use this file except in compliance with the License. You may obtain
3
+# a copy of the License at
4
+#
5
+#      http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+# License for the specific language governing permissions and limitations
11
+# under the License.
12
+
13
+import re
14
+
15
+from keystonemiddleware import auth_token
16
+from oslo_log import log
17
+
18
+from gyan.common import exception
19
+from gyan.common.i18n import _
20
+from gyan.common import utils
21
+
22
+LOG = log.getLogger(__name__)
23
+
24
+
25
+class AuthTokenMiddleware(auth_token.AuthProtocol):
26
+    """A wrapper on Keystone auth_token middleware.
27
+
28
+    Does not perform verification of authentication tokens
29
+    for public routes in the API.
30
+
31
+    """
32
+
33
+    def __init__(self, app, conf, public_api_routes=None):
34
+        if public_api_routes is None:
35
+            public_api_routes = []
36
+        route_pattern_tpl = '%s(\.json)?$'
37
+
38
+        try:
39
+            self.public_api_routes = [re.compile(route_pattern_tpl % route_tpl)
40
+                                      for route_tpl in public_api_routes]
41
+        except re.error as e:
42
+            msg = _('Cannot compile public API routes: %s') % e
43
+
44
+            LOG.error(msg)
45
+            raise exception.ConfigInvalid(error_msg=msg)
46
+
47
+        super(AuthTokenMiddleware, self).__init__(app, conf)
48
+
49
+    def __call__(self, env, start_response):
50
+        path = utils.safe_rstrip(env.get('PATH_INFO'), '/')
51
+
52
+        # The information whether the API call is being performed against the
53
+        # public API is required for some other components. Saving it to the
54
+        # WSGI environment is reasonable thereby.
55
+        env['is_public_api'] = any([re.match(pattern, path)
56
+                                    for pattern in self.public_api_routes])
57
+
58
+        if env['is_public_api']:
59
+            return self._app(env, start_response)
60
+
61
+        return super(AuthTokenMiddleware, self).__call__(env, start_response)
62
+
63
+    @classmethod
64
+    def factory(cls, global_config, **local_conf):
65
+        public_routes = local_conf.get('acl_public_routes', '')
66
+        public_api_routes = [path.strip() for path in public_routes.split(',')]
67
+
68
+        def _factory(app):
69
+            return cls(app, global_config, public_api_routes=public_api_routes)
70
+
71
+        return _factory

+ 99
- 0
gyan/api/middleware/parsable_error.py View File

@@ -0,0 +1,99 @@
1
+# Copyright ? 2012 New Dream Network, LLC (DreamHost)
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+# not use this file except in compliance with the License. You may obtain
5
+# a copy of the License at
6
+#
7
+#      http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+# License for the specific language governing permissions and limitations
13
+# under the License.
14
+"""
15
+Middleware to replace the plain text message body of an error
16
+response with one formatted so the client can parse it.
17
+
18
+Based on pecan.middleware.errordocument
19
+"""
20
+
21
+import six
22
+
23
+from oslo_serialization import jsonutils as json
24
+from gyan.common.i18n import _
25
+
26
+
27
+class ParsableErrorMiddleware(object):
28
+    """Replace error body with something the client can parse."""
29
+
30
+    def __init__(self, app):
31
+        self.app = app
32
+
33
+    def __call__(self, environ, start_response):
34
+        # Request for this state, modified by replace_start_response()
35
+        # and used when an error is being reported.
36
+        state = {}
37
+
38
+        def replacement_start_response(status, headers, exc_info=None):
39
+            """Overrides the default response to make errors parsable."""
40
+            try:
41
+                status_code = int(status.split(' ')[0])
42
+                state['status_code'] = status_code
43
+            except (ValueError, TypeError):  # pragma: nocover
44
+                raise Exception(_(
45
+                    'ErrorDocumentMiddleware received an invalid '
46
+                    'status %s') % status)
47
+            else:
48
+                if (state['status_code'] // 100) not in (2, 3):
49
+                    # Remove some headers so we can replace them later
50
+                    # when we have the full error message and can
51
+                    # compute the length.
52
+                    headers = [(h, v)
53
+                               for (h, v) in headers
54
+                               if h not in ('Content-Length', 'Content-Type')
55
+                               ]
56
+                # Save the headers in case we need to modify them.
57
+                state['headers'] = headers
58
+                return start_response(status, headers, exc_info)
59
+
60
+        app_iter = self.app(environ, replacement_start_response)
61
+
62
+        if (state['status_code'] // 100) not in (2, 3):
63
+            errs = []
64
+            for err_str in app_iter:
65
+                err = {}
66
+                try:
67
+                    err = json.loads(err_str.decode('utf-8'))
68
+                except ValueError:
69
+                    pass
70
+
71
+                if 'title' in err and 'description' in err:
72
+                    title = err['title']
73
+                    desc = err['description']
74
+                else:
75
+                    title = ''
76
+                    desc = ''
77
+
78
+                error_code = err['faultstring'].lower() \
79
+                    if 'faultstring' in err else ''
80
+                # 'ml-infra' is the service-name. The general form of the
81
+                # code is service-name.error-code.
82
+                code = '.'.join(['ml-infra', error_code])
83
+
84
+                errs.append({
85
+                    'request_id': '',
86
+                    'code': code,
87
+                    'status': state['status_code'],
88
+                    'title': title,
89
+                    'detail': desc,
90
+                    'links': []
91
+                })
92
+
93
+            body = [six.b(json.dumps({'errors': errs}))]
94
+
95