From c7b24fda7fbf2c117bcd061be2c6dd9ae88acd56 Mon Sep 17 00:00:00 2001 From: zhuli Date: Sat, 29 Jul 2017 11:51:22 +0800 Subject: [PATCH] Add cyborg-api command Change-Id: I392ad0383d5d8299db407df79bd60adbfbd428cd --- cyborg/cmd/__init__.py | 19 ++++++ cyborg/cmd/api.py | 36 +++++++++++ cyborg/common/__init__.py | 0 cyborg/common/config.py | 26 ++++++++ cyborg/common/exception.py | 90 +++++++++++++++++++++++++++ cyborg/common/i18n.py | 22 +++++++ cyborg/common/rpc.py | 121 +++++++++++++++++++++++++++++++++++++ cyborg/common/service.py | 98 ++++++++++++++++++++++++++++++ cyborg/conf/__init__.py | 25 ++++++++ cyborg/conf/api.py | 49 +++++++++++++++ cyborg/conf/default.py | 32 ++++++++++ cyborg/version.py | 19 ++++++ requirements.txt | 9 +++ setup.cfg | 4 ++ 14 files changed, 550 insertions(+) create mode 100644 cyborg/cmd/__init__.py create mode 100644 cyborg/cmd/api.py create mode 100644 cyborg/common/__init__.py create mode 100644 cyborg/common/config.py create mode 100644 cyborg/common/exception.py create mode 100644 cyborg/common/i18n.py create mode 100644 cyborg/common/rpc.py create mode 100644 cyborg/common/service.py create mode 100644 cyborg/conf/__init__.py create mode 100644 cyborg/conf/api.py create mode 100644 cyborg/conf/default.py create mode 100644 cyborg/version.py diff --git a/cyborg/cmd/__init__.py b/cyborg/cmd/__init__.py new file mode 100644 index 00000000..b90e7b3b --- /dev/null +++ b/cyborg/cmd/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import oslo_i18n as i18n + + +i18n.install('cyborg') diff --git a/cyborg/cmd/api.py b/cyborg/cmd/api.py new file mode 100644 index 00000000..7199e7b6 --- /dev/null +++ b/cyborg/cmd/api.py @@ -0,0 +1,36 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""The Cyborg Service API.""" + +import sys + +from oslo_config import cfg + +from cyborg.common import service as cyborg_service + + +CONF = cfg.CONF + + +def main(): + # Parse config file and command line options, then start logging + cyborg_service.prepare_service(sys.argv) + + # Build and start the WSGI app + launcher = cyborg_service.process_launcher() + server = cyborg_service.WSGIService('cyborg_api', CONF.api.enable_ssl_api) + launcher.launch_service(server, workers=server.workers) + launcher.wait() diff --git a/cyborg/common/__init__.py b/cyborg/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cyborg/common/config.py b/cyborg/common/config.py new file mode 100644 index 00000000..29ba17ad --- /dev/null +++ b/cyborg/common/config.py @@ -0,0 +1,26 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from cyborg import version + + +def parse_args(argv, default_config_files=None): + version_string = version.version_info.release_string() + cfg.CONF(argv[1:], + project='cyborg', + version=version_string, + default_config_files=default_config_files) diff --git a/cyborg/common/exception.py b/cyborg/common/exception.py new file mode 100644 index 00000000..e077e6a8 --- /dev/null +++ b/cyborg/common/exception.py @@ -0,0 +1,90 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Cyborg base exception handling. + +SHOULD include dedicated exception logging. + +""" + +from oslo_log import log +import six +from six.moves import http_client + +from cyborg.common.i18n import _ +from cyborg.conf import CONF + + +LOG = log.getLogger(__name__) + + +class CyborgException(Exception): + """Base Cyborg Exception + + To correctly use this class, inherit from it and define + a '_msg_fmt' property. That message will get printf'd + with the keyword arguments provided to the constructor. + + If you need to access the message from an exception you should use + six.text_type(exc) + + """ + _msg_fmt = _("An unknown exception occurred.") + code = http_client.INTERNAL_SERVER_ERROR + headers = {} + safe = False + + def __init__(self, message=None, **kwargs): + self.kwargs = kwargs + + if 'code' not in self.kwargs: + try: + self.kwargs['code'] = self.code + except AttributeError: + pass + + if not message: + try: + message = self._msg_fmt % kwargs + except Exception: + # kwargs doesn't match a variable in self._msg_fmt + # log the issue and the kwargs + LOG.exception('Exception in string format operation') + for name, value in kwargs.items(): + LOG.error("%s: %s" % (name, value)) + + if CONF.fatal_exception_format_errors: + raise + else: + # at least get the core self._msg_fmt out if something + # happened + message = self._msg_fmt + + super(CyborgException, self).__init__(message) + + def __str__(self): + """Encode to utf-8 then wsme api can consume it as well.""" + if not six.PY3: + return unicode(self.args[0]).encode('utf-8') + + return self.args[0] + + def __unicode__(self): + """Return a unicode representation of the exception message.""" + return unicode(self.args[0]) + + +class ConfigInvalid(CyborgException): + _msg_fmt = _("Invalid configuration file. %(error_msg)s") diff --git a/cyborg/common/i18n.py b/cyborg/common/i18n.py new file mode 100644 index 00000000..eb5c3139 --- /dev/null +++ b/cyborg/common/i18n.py @@ -0,0 +1,22 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import oslo_i18n as i18n + + +_translators = i18n.TranslatorFactory(domain='cyborg') + +# The primary translation function using the well-known name "_" +_ = _translators.primary diff --git a/cyborg/common/rpc.py b/cyborg/common/rpc.py new file mode 100644 index 00000000..4947d76f --- /dev/null +++ b/cyborg/common/rpc.py @@ -0,0 +1,121 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from oslo_context import context as cyborg_context +import oslo_messaging as messaging +from oslo_messaging.rpc import dispatcher + +from cyborg.common import exception + + +CONF = cfg.CONF +TRANSPORT = None +NOTIFICATION_TRANSPORT = None +NOTIFIER = None + +ALLOWED_EXMODS = [ + exception.__name__, +] +EXTRA_EXMODS = [] + + +def init(conf): + global TRANSPORT, NOTIFICATION_TRANSPORT, NOTIFIER + exmods = get_allowed_exmods() + TRANSPORT = messaging.get_rpc_transport(conf, + allowed_remote_exmods=exmods) + NOTIFICATION_TRANSPORT = messaging.get_notification_transport( + conf, + allowed_remote_exmods=exmods) + serializer = RequestContextSerializer(messaging.JsonPayloadSerializer()) + NOTIFIER = messaging.Notifier(NOTIFICATION_TRANSPORT, + serializer=serializer, + topics=['notifications']) + + +def cleanup(): + global TRANSPORT, NOTIFICATION_TRANSPORT, NOTIFIER + assert TRANSPORT is not None + assert NOTIFICATION_TRANSPORT is not None + assert NOTIFIER is not None + TRANSPORT.cleanup() + NOTIFICATION_TRANSPORT.cleanup() + TRANSPORT = NOTIFICATION_TRANSPORT = NOTIFIER = None + + +def set_defaults(control_exchange): + messaging.set_transport_defaults(control_exchange) + + +def add_extra_exmods(*args): + EXTRA_EXMODS.extend(args) + + +def clear_extra_exmods(): + del EXTRA_EXMODS[:] + + +def get_allowed_exmods(): + return ALLOWED_EXMODS + EXTRA_EXMODS + + +class RequestContextSerializer(messaging.Serializer): + def __init__(self, base): + self._base = base + + def serialize_entity(self, context, entity): + if not self._base: + return entity + return self._base.serialize_entity(context, entity) + + def deserialize_entity(self, context, entity): + if not self._base: + return entity + return self._base.deserialize_entity(context, entity) + + def serialize_context(self, context): + return context.to_dict() + + def deserialize_context(self, context): + return cyborg_context.RequestContext.from_dict(context) + + +def get_client(target, version_cap=None, serializer=None): + assert TRANSPORT is not None + serializer = RequestContextSerializer(serializer) + return messaging.RPCClient(TRANSPORT, + target, + version_cap=version_cap, + serializer=serializer) + + +def get_server(target, endpoints, serializer=None): + assert TRANSPORT is not None + access_policy = dispatcher.DefaultRPCAccessPolicy + serializer = RequestContextSerializer(serializer) + return messaging.get_rpc_server(TRANSPORT, + target, + endpoints, + executor='eventlet', + serializer=serializer, + access_policy=access_policy) + + +def get_notifier(service=None, host=None, publisher_id=None): + assert NOTIFIER is not None + if not publisher_id: + publisher_id = "%s.%s" % (service, host or CONF.host) + return NOTIFIER.prepare(publisher_id=publisher_id) diff --git a/cyborg/common/service.py b/cyborg/common/service.py new file mode 100644 index 00000000..49a9ba30 --- /dev/null +++ b/cyborg/common/service.py @@ -0,0 +1,98 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_concurrency import processutils +from oslo_log import log +from oslo_service import service +from oslo_service import wsgi + +from cyborg.api import app +from cyborg.common import config +from cyborg.common import exception +from cyborg.common.i18n import _ +from cyborg.common import rpc +from cyborg.conf import CONF + + +LOG = log.getLogger(__name__) + + +def prepare_service(argv=None): + log.register_options(CONF) + log.set_defaults() + rpc.set_defaults(control_exchange='cyborg') + + argv = argv or [] + config.parse_args(argv) + + log.setup(CONF, 'cyborg') + rpc.init(CONF) + + +def process_launcher(): + return service.ProcessLauncher(CONF) + + +class WSGIService(service.ServiceBase): + """Provides ability to launch cyborg API from wsgi app.""" + + def __init__(self, name, use_ssl=False): + """Initialize, but do not start the WSGI server. + + :param name: The name of the WSGI server given to the loader. + :param use_ssl: Wraps the socket in an SSL context if True. + :returns: None + """ + self.name = name + self.app = app.setup_app() + self.workers = (CONF.api.api_workers or + processutils.get_worker_count()) + if self.workers and self.workers < 1: + raise exception.ConfigInvalid( + _("api_workers value of %d is invalid, " + "must be greater than 0.") % self.workers) + + self.server = wsgi.Server(CONF, self.name, self.app, + host=CONF.api.host_ip, + port=CONF.api.port, + use_ssl=use_ssl) + + def start(self): + """Start serving this service using loaded configuration. + + :returns: None + """ + self.server.start() + + def stop(self): + """Stop serving this API. + + :returns: None + """ + self.server.stop() + + def wait(self): + """Wait for the service to stop serving this API. + + :returns: None + """ + self.server.wait() + + def reset(self): + """Reset server greenpool size to default. + + :returns: None + """ + self.server.reset() diff --git a/cyborg/conf/__init__.py b/cyborg/conf/__init__.py new file mode 100644 index 00000000..448a3f8a --- /dev/null +++ b/cyborg/conf/__init__.py @@ -0,0 +1,25 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from cyborg.conf import api +from cyborg.conf import default + + +CONF = cfg.CONF + +api.register_opts(CONF) +default.register_opts(CONF) diff --git a/cyborg/conf/api.py b/cyborg/conf/api.py new file mode 100644 index 00000000..cbfeede9 --- /dev/null +++ b/cyborg/conf/api.py @@ -0,0 +1,49 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from cyborg.common.i18n import _ + + +opts = [ + cfg.HostAddressOpt('host_ip', + default='0.0.0.0', + help=_('The IP address on which cyborg-api listens.')), + cfg.PortOpt('port', + default=6666, + help=_('The TCP port on which cyborg-api listens.')), + cfg.IntOpt('api_workers', + help=_('Number of workers for OpenStack Cyborg API service. ' + 'The default is equal to the number of CPUs available ' + 'if that can be determined, else a default worker ' + 'count of 1 is returned.')), + cfg.BoolOpt('enable_ssl_api', + default=False, + help=_("Enable the integrated stand-alone API to service " + "requests via HTTPS instead of HTTP. If there is a " + "front-end service performing HTTPS offloading from " + "the service, this option should be False; note, you " + "will want to change public API endpoint to represent " + "SSL termination URL with 'public_endpoint' option.")), +] + +opt_group = cfg.OptGroup(name='api', + title='Options for the cyborg-api service') + + +def register_opts(conf): + conf.register_group(opt_group) + conf.register_opts(opts, group=opt_group) diff --git a/cyborg/conf/default.py b/cyborg/conf/default.py new file mode 100644 index 00000000..e248ca5c --- /dev/null +++ b/cyborg/conf/default.py @@ -0,0 +1,32 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from cyborg.common.i18n import _ + + +exc_log_opts = [ + cfg.BoolOpt('fatal_exception_format_errors', + default=False, + help=_('Used if there is a formatting error when generating ' + 'an exception message (a programming error). If True, ' + 'raise an exception; if False, use the unformatted ' + 'message.')), +] + + +def register_opts(conf): + conf.register_opts(exc_log_opts) diff --git a/cyborg/version.py b/cyborg/version.py new file mode 100644 index 00000000..fd1dcdc6 --- /dev/null +++ b/cyborg/version.py @@ -0,0 +1,19 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pbr.version + + +version_info = pbr.version.VersionInfo('cyborg') diff --git a/requirements.txt b/requirements.txt index 3934ced3..83679503 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,12 @@ pbr!=2.1.0,>=2.0.0 # Apache-2.0 pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.0.0 # BSD WSME>=0.8 # MIT six>=1.9.0 # MIT +eventlet!=0.18.3,!=0.20.1,<0.21.0,>=0.18.2 # MIT +oslo.i18n!=3.15.2,>=2.1.0 # Apache-2.0 +oslo.config!=4.3.0,!=4.4.0,>=4.0.0 # Apache-2.0 +oslo.log>=3.22.0 # Apache-2.0 +oslo.context>=2.14.0 # Apache-2.0 +oslo.messaging!=5.25.0,>=5.24.2 # Apache-2.0 +oslo.concurrency>=3.8.0 # Apache-2.0 +oslo.service>=1.10.0 # Apache-2.0 +oslo.serialization!=2.19.1,>=1.10.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index 6d27ab5c..4dab9ec3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,6 +23,10 @@ classifier = packages = cyborg +[entry_points] +console_scripts = + cyborg-api = cyborg.cmd.api:main + [build_sphinx] source-dir = doc/source build-dir = doc/build