From 814b03041712c618c42478fd1fcb86664b945a59 Mon Sep 17 00:00:00 2001 From: Nikolay Mahotkin Date: Mon, 16 Dec 2013 17:21:09 +0400 Subject: [PATCH] Add demo app for Mistral * Simple server * expose one endpoint - /tasks * recieve correct signal from mistral about task execution, log 'task N started, task N finished' * logging Change-Id: I230eac12319be820d64612d242b9190a10be946e --- mistral-demo-app/README.md | 24 ++++++ mistral-demo-app/demo_app/__init__.py | 0 mistral-demo-app/demo_app/api/__init__.py | 0 mistral-demo-app/demo_app/api/app.py | 40 ++++++++++ mistral-demo-app/demo_app/api/client.py | 58 ++++++++++++++ .../demo_app/api/controllers/__init__.py | 0 .../demo_app/api/controllers/resource.py | 54 +++++++++++++ .../demo_app/api/controllers/root.py | 71 +++++++++++++++++ .../demo_app/api/controllers/tasks.py | 70 +++++++++++++++++ mistral-demo-app/demo_app/cmd/__init__.py | 0 mistral-demo-app/demo_app/cmd/main.py | 78 +++++++++++++++++++ mistral-demo-app/demo_app/config.py | 36 +++++++++ mistral-demo-app/demo_app/demo.yaml | 57 ++++++++++++++ mistral-demo-app/demo_app/tasks.py | 37 +++++++++ mistral-demo-app/demo_app/version.py | 19 +++++ mistral-demo-app/requirements.txt | 10 +++ mistral-demo-app/setup.cfg | 27 +++++++ mistral-demo-app/setup.py | 21 +++++ mistral-demo-app/tox.ini | 42 ++++++++++ 19 files changed, 644 insertions(+) create mode 100644 mistral-demo-app/README.md create mode 100644 mistral-demo-app/demo_app/__init__.py create mode 100644 mistral-demo-app/demo_app/api/__init__.py create mode 100644 mistral-demo-app/demo_app/api/app.py create mode 100644 mistral-demo-app/demo_app/api/client.py create mode 100644 mistral-demo-app/demo_app/api/controllers/__init__.py create mode 100644 mistral-demo-app/demo_app/api/controllers/resource.py create mode 100644 mistral-demo-app/demo_app/api/controllers/root.py create mode 100644 mistral-demo-app/demo_app/api/controllers/tasks.py create mode 100644 mistral-demo-app/demo_app/cmd/__init__.py create mode 100644 mistral-demo-app/demo_app/cmd/main.py create mode 100644 mistral-demo-app/demo_app/config.py create mode 100644 mistral-demo-app/demo_app/demo.yaml create mode 100644 mistral-demo-app/demo_app/tasks.py create mode 100644 mistral-demo-app/demo_app/version.py create mode 100644 mistral-demo-app/requirements.txt create mode 100644 mistral-demo-app/setup.cfg create mode 100644 mistral-demo-app/setup.py create mode 100644 mistral-demo-app/tox.ini diff --git a/mistral-demo-app/README.md b/mistral-demo-app/README.md new file mode 100644 index 00000000..a929209b --- /dev/null +++ b/mistral-demo-app/README.md @@ -0,0 +1,24 @@ +Mistral Demo app +================ + +This mini-project demonstrates basic Mistral capabilities. + + +### Installation +First of all, in a shell run: + +*tox* + +This will install necessary virtual environments and run all the project tests. Installing virtual environments may take significant time (~10-15 mins). + +Then make a sym-link to mistralclient package + +*cd mistral-demo-app* +*ln -s mistralclient + +### Running Mistral Demo app server +To run Mistral Demo app server perform the following commands in a shell: + +*tox -evenv -- python demo_app/cmd/main.py* + +Then it will automatically upload necessary workbook and run workflow. diff --git a/mistral-demo-app/demo_app/__init__.py b/mistral-demo-app/demo_app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mistral-demo-app/demo_app/api/__init__.py b/mistral-demo-app/demo_app/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mistral-demo-app/demo_app/api/app.py b/mistral-demo-app/demo_app/api/app.py new file mode 100644 index 00000000..b5fd518e --- /dev/null +++ b/mistral-demo-app/demo_app/api/app.py @@ -0,0 +1,40 @@ +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pecan + + +app = { + 'root': 'demo_app.api.controllers.root.RootController', + 'modules': ['demo_app.api'], + 'debug': True, +} + + +def get_pecan_config(): + # Set up the pecan configuration + return pecan.configuration.conf_from_dict(app) + + +def setup_app(config=None): + if not config: + config = get_pecan_config() + + app_conf = dict(config) + + return pecan.make_app( + app_conf.pop('root'), + logging=getattr(config, 'logging', {}), + **app_conf + ) diff --git a/mistral-demo-app/demo_app/api/client.py b/mistral-demo-app/demo_app/api/client.py new file mode 100644 index 00000000..1cd6a3a4 --- /dev/null +++ b/mistral-demo-app/demo_app/api/client.py @@ -0,0 +1,58 @@ +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import pkg_resources as pkg + +from mistralclient.api import client + +from demo_app import version + +MISTRAL_URL = "http://localhost:8989/v1" +client.Client.authenticate = mock.MagicMock(return_value=(MISTRAL_URL, + "", "", "")) +CLIENT = client.Client(mistral_url=MISTRAL_URL, + project_name="mistral_demo") + + +WB_NAME = "myWorkbook" +TARGET_TASK = "task4" + + +def upload_workbook(): + try: + CLIENT.workbooks.get(WB_NAME) + except: + CLIENT.workbooks.create(WB_NAME, + description="My test workbook", + tags=["test"]) + print("Uploading workbook definition...\n") + definition = get_workbook_definition() + CLIENT.workbooks.upload_definition(WB_NAME, definition) + print definition + print("\nUploaded.") + + +def get_workbook_definition(): + return open(pkg.resource_filename(version.version_info.package, + "demo.yaml")).read() + + +def start_execution(): + import threading + t = threading.Thread(target=CLIENT.executions.create, + kwargs={'workbook_name': WB_NAME, + 'target_task': TARGET_TASK}) + t.start() + return "accepted" diff --git a/mistral-demo-app/demo_app/api/controllers/__init__.py b/mistral-demo-app/demo_app/api/controllers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mistral-demo-app/demo_app/api/controllers/resource.py b/mistral-demo-app/demo_app/api/controllers/resource.py new file mode 100644 index 00000000..2ee83957 --- /dev/null +++ b/mistral-demo-app/demo_app/api/controllers/resource.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from wsme import types as wtypes + + +LOG = logging.getLogger(__name__) + +API_STATUS = wtypes.Enum(str, 'SUPPORTED', 'CURRENT', 'DEPRECATED') + + +class Resource(wtypes.Base): + """REST API Resource.""" + + @classmethod + def from_dict(cls, d): + # TODO: take care of nested resources + obj = cls() + + for key, val in d.items(): + if hasattr(obj, key): + setattr(obj, key, val) + + return obj + + def __str__(self): + """WSME based implementation of __str__.""" + + res = "%s [" % type(self).__name__ + + first = True + for attr in self._wsme_attributes: + if not first: + res += ', ' + else: + first = False + + res += "%s='%s'" % (attr.name, getattr(self, attr.name)) + + return res + "]" diff --git a/mistral-demo-app/demo_app/api/controllers/root.py b/mistral-demo-app/demo_app/api/controllers/root.py new file mode 100644 index 00000000..ce67feb7 --- /dev/null +++ b/mistral-demo-app/demo_app/api/controllers/root.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import pecan +from pecan import rest +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from demo_app.api.controllers import resource +from demo_app.api.controllers import tasks +from demo_app.api import client + +LOG = logging.getLogger(__name__) + +API_STATUS = wtypes.Enum(str, 'SUPPORTED', 'CURRENT', 'DEPRECATED') + + +class Link(resource.Resource): + """Web link.""" + + href = wtypes.text + target = wtypes.text + + +class APIVersion(resource.Resource): + """API Version.""" + + id = wtypes.text + status = API_STATUS + link = Link + + +class StartController(rest.RestController): + @wsme_pecan.wsexpose(wtypes.text) + def get(self): + print("Start execution for: %s" % client.TARGET_TASK) + + execution = client.start_execution() + + return execution + + +class RootController(object): + + tasks = tasks.Controller() + start = StartController() + + @wsme_pecan.wsexpose([APIVersion]) + def index(self): + LOG.debug("Fetching API versions.") + + host_url = '%s/%s' % (pecan.request.host_url, 'v1') + api_v1 = APIVersion(id='v1.0', + status='CURRENT', + link=Link(href=host_url, target='v1')) + + return [api_v1] diff --git a/mistral-demo-app/demo_app/api/controllers/tasks.py b/mistral-demo-app/demo_app/api/controllers/tasks.py new file mode 100644 index 00000000..2c2433ae --- /dev/null +++ b/mistral-demo-app/demo_app/api/controllers/tasks.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from pecan import abort +from pecan import rest +import pecan +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from demo_app import tasks + + +LOG = logging.getLogger(__name__) + + +class Controller(rest.RestController): + """API root controller""" + + @wsme_pecan.wsexpose(wtypes.text) + def get_all(self): + LOG.debug("Fetch items.") + + values = { + 'tasks': [ + 'task1', + 'task2', + 'task3', + 'task4' + ] + } + + if not values: + abort(404) + else: + return values + + @wsme_pecan.wsexpose(wtypes.text, wtypes.text) + def get(self, name): + print("Task '%s' is starting" % name) + + value = "Task %s accepted" % name + tasks.start_task(**self.get_mistral_headers()) + return value + + def get_mistral_headers(self): + headers = pecan.request.headers + try: + needed_headers = { + 'workbook_name': headers['Mistral-Workbook-Name'], + 'execution_id': headers['Mistral-Execution-Id'], + 'task_id': headers['Mistral-Task-Id'] + } + return needed_headers + except KeyError: + raise RuntimeError("Could not find http headers for " + "defining mistral task") diff --git a/mistral-demo-app/demo_app/cmd/__init__.py b/mistral-demo-app/demo_app/cmd/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mistral-demo-app/demo_app/cmd/main.py b/mistral-demo-app/demo_app/cmd/main.py new file mode 100644 index 00000000..11d48639 --- /dev/null +++ b/mistral-demo-app/demo_app/cmd/main.py @@ -0,0 +1,78 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Script to start Demo API service.""" + +import eventlet + +import logging +import os +from requests import exceptions +import sys +import threading +from time import sleep +from wsgiref import simple_server + +from oslo.config import cfg + +from demo_app import config +from demo_app.api import app +from demo_app.api import client + + +eventlet.monkey_patch( + os=True, select=True, socket=True, thread=True, time=True) + +logging.basicConfig(level=logging.WARN) +LOG = logging.getLogger('demo_app.cmd.main') +CLIENT = client.CLIENT + + +def upload_wb_and_start(): + sleep(5) + try: + client.upload_workbook() + except exceptions.ConnectionError: + LOG.error("Error. Mistral service probably is not working now") + sys.exit(1) + print("Start execution for: %s" % client.TARGET_TASK) + + client.start_execution() + + +def main(): + try: + config.parse_args() + + host = cfg.CONF.api.host + port = cfg.CONF.api.port + + server = simple_server.make_server(host, port, app.setup_app()) + + LOG.info("Demo app API is serving on http://%s:%s (PID=%s)" % + (host, port, os.getpid())) + + server.serve_forever() + except RuntimeError, e: + sys.stderr.write("ERROR: %s\n" % e) + sys.exit(1) + + +if __name__ == '__main__': + upload_thread = threading.Thread(target=upload_wb_and_start) + upload_thread.run() + main() diff --git a/mistral-demo-app/demo_app/config.py b/mistral-demo-app/demo_app/config.py new file mode 100644 index 00000000..069672a5 --- /dev/null +++ b/mistral-demo-app/demo_app/config.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo.config import cfg + +from demo_app import version + + +api_opts = [ + cfg.StrOpt('host', default='0.0.0.0', help='Demo-app API server host'), + cfg.IntOpt('port', default=8988, help='Demo-app API server port') +] + +CONF = cfg.CONF +CONF.register_opts(api_opts, group='api') + + +def parse_args(args=None, usage=None, default_config_files=None): + CONF(args=args, + project='mistral-demo', + version=version, + usage=usage, + default_config_files=default_config_files) diff --git a/mistral-demo-app/demo_app/demo.yaml b/mistral-demo-app/demo_app/demo.yaml new file mode 100644 index 00000000..939897df --- /dev/null +++ b/mistral-demo-app/demo_app/demo.yaml @@ -0,0 +1,57 @@ +Services: + MyRest: + type: REST_API + parameters: + baseUrl: http://localhost:8988 + actions: + task1: + parameters: + url: /tasks/task1 + method: GET + task-parameters: + + task2: + parameters: + url: /tasks/task2 + method: GET + task-parameters: + + task3: + parameters: + url: /tasks/task3 + method: GET + task-parameters: + + task4: + parameters: + url: /tasks/task4 + method: GET + task-parameters: + +Workflow: + tasks: + task1: + action: MyRest:task1 + parameters: + + task2: + dependsOn: [task1] + action: MyRest:task2 + parameters: + + task3: + dependsOn: [task1] + action: MyRest:task3 + parameters: + + task4: + dependsOn: [task2, task3] + action: MyRest:task4 + parameters: + + events: + task4: + type: periodic + tasks: task4 + parameters: + cron-pattern: "*/1 * * * *" diff --git a/mistral-demo-app/demo_app/tasks.py b/mistral-demo-app/demo_app/tasks.py new file mode 100644 index 00000000..5ae6a6fb --- /dev/null +++ b/mistral-demo-app/demo_app/tasks.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from time import sleep +import threading + +from demo_app.api import client + + +CLIENT = client.CLIENT + + +def start_task(**kwargs): + thread = threading.Thread(target=finish_task, kwargs=kwargs) + thread.start() + + +def finish_task(task_id, execution_id, workbook_name): + # simulate working + sleep(8) + + task = CLIENT.tasks.update(workbook_name, execution_id, + task_id, "SUCCESS") + print("Task %s - SUCCESS" % task.name) diff --git a/mistral-demo-app/demo_app/version.py b/mistral-demo-app/demo_app/version.py new file mode 100644 index 00000000..d72d602e --- /dev/null +++ b/mistral-demo-app/demo_app/version.py @@ -0,0 +1,19 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pbr import version + +version_info = version.VersionInfo('demo_app') +version_string = version_info.version_string diff --git a/mistral-demo-app/requirements.txt b/mistral-demo-app/requirements.txt new file mode 100644 index 00000000..94c81022 --- /dev/null +++ b/mistral-demo-app/requirements.txt @@ -0,0 +1,10 @@ +pbr>=0.5.21,<1.0 +eventlet>=0.13.0 +pyyaml +mock +pecan>=0.2.0 +WSME>=0.5b6 +amqplib>=0.6.1 +argparse +oslo.config>=1.2.0 +python-keystoneclient diff --git a/mistral-demo-app/setup.cfg b/mistral-demo-app/setup.cfg new file mode 100644 index 00000000..8c352ab3 --- /dev/null +++ b/mistral-demo-app/setup.cfg @@ -0,0 +1,27 @@ +[metadata] +name = demo-app +version = 0.01 +summary = Demo-app from Mistral +description-file = README.rd +#license = Apache Software License +classifiers = + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Environment :: OpenStack + Intended Audience :: Information Technology + Intended Audience :: System Administrators + #License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux +author = OpenStack +author-email = openstack-dev@lists.openstack.org + +[files] +packages = + demo_app + +[nosetests] +cover-package = demo_app + +[extract_messages] +keywords = _ gettext ngettext l_ lazy_gettext diff --git a/mistral-demo-app/setup.py b/mistral-demo-app/setup.py new file mode 100644 index 00000000..e47a6d81 --- /dev/null +++ b/mistral-demo-app/setup.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT +import setuptools + +setuptools.setup( + name="demo_app") diff --git a/mistral-demo-app/tox.ini b/mistral-demo-app/tox.ini new file mode 100644 index 00000000..7423a688 --- /dev/null +++ b/mistral-demo-app/tox.ini @@ -0,0 +1,42 @@ +[tox] +envlist = py26,py27,py33,pep8 +minversion = 1.6 +skipsdist = True + +[testenv] +usedevelop = True +install_command = pip install -U {opts} {packages} +setenv = + VIRTUAL_ENV={envdir} + NOSE_WITH_OPENSTACK=1 + NOSE_OPENSTACK_COLOR=1 + NOSE_OPENSTACK_RED=0.05 + NOSE_OPENSTACK_YELLOW=0.025 + NOSE_OPENSTACK_SHOW_ELAPSED=1 + NOSE_OPENSTACK_STDOUT=1 + NOSE_XUNIT=1 +deps = + -r{toxinidir}/requirements.txt +commands = nosetests + +[testenv:pep8] +commands = flake8 {posargs} + +[testenv:venv] +commands = {posargs} + +[testenv:docs] +commands = + rm -rf doc/html doc/build + rm -rf doc/source/apidoc doc/source/api + python setup.py build_sphinx + +[testenv:pylint] +setenv = VIRTUAL_ENV={envdir} +commands = bash tools/lintstack.sh + +[flake8] +show-source = true +builtins = _ +exclude=.venv,.git,.tox,*egg,tools +