diff --git a/doc/source/index.rst b/doc/source/index.rst index 405d808d9..4087550ba 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -126,8 +126,8 @@ In order to keep these layers decoupled, we have established that transport drivers must guarantee that the incoming data is well-formed and storage drivers must enforce their data model stays consistent. -Setting up a development environment -==================================== +Setting up Zaqar in development environment +=========================================== .. toctree:: :maxdepth: 1 @@ -144,7 +144,6 @@ Welcome new contributors first_patch first_review - Running and writing tests ========================= diff --git a/install-guide/source/conf.py b/install-guide/source/conf.py new file mode 100644 index 000000000..fbd85f08d --- /dev/null +++ b/install-guide/source/conf.py @@ -0,0 +1,301 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import os +# import sys + + +import openstackdocstheme + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +# TODO(ajaeger): enable PDF building, for example add 'rst2pdf.pdfbuilder' +# extensions = + +# Add any paths that contain templates here, relative to this directory. +# templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Installation Guide for Messaging Service' +bug_tag = u'install-guide' +copyright = u'2016, OpenStack contributors' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.1' +# The full version, including alpha/beta/rc tags. +release = '0.1' + +# A few variables have to be set for the log-a-bug feature. +# giturl: The location of conf.py on Git. Must be set manually. +# gitsha: The SHA checksum of the bug description. Automatically extracted +# from git log. +# bug_tag: Tag for categorizing the bug. Must be set manually. +# These variables are passed to the logabug code via html_context. +giturl = (u'http://git.openstack.org/cgit/openstack/zaqar/tree/install-guide' + u'/source') +git_cmd = "/usr/bin/git log | head -n1 | cut -f2 -d' '" +gitsha = os.popen(git_cmd).read().strip('\n') +html_context = {"gitsha": gitsha, "bug_tag": bug_tag, + "giturl": giturl} + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'openstackdocs' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +html_theme_path = [openstackdocstheme.get_html_theme_path()] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = [] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# So that we can enable "log-a-bug" links from each output HTML page, this +# variable must be set to a format that includes year, month, day, hours and +# minutes. +html_last_updated_fmt = '%Y-%m-%d %H:%M' + + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +html_use_index = False + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +html_show_sourcelink = False + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'install-guide' + +# If true, publish source files +html_copy_source = False + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # 'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', 'InstallGuide.tex', u'Install Guide', + u'OpenStack contributors', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'installguide', u'Install Guide', + [u'OpenStack contributors'], 1) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'InstallGuide', u'Install Guide', + u'OpenStack contributors', 'InstallGuide', + 'This guide shows OpenStack end users how to install ' + 'an OpenStack cloud.', 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# texinfo_no_detailmenu = False + +# -- Options for Internationalization output ------------------------------ +locale_dirs = ['locale/'] + +# -- Options for PDF output -------------------------------------------------- + +pdf_documents = [ + ('index', u'InstallGuide', u'Install Guide', + u'OpenStack contributors') +] diff --git a/install-guide/source/get_started.rst b/install-guide/source/get_started.rst new file mode 100644 index 000000000..8b30d02e0 --- /dev/null +++ b/install-guide/source/get_started.rst @@ -0,0 +1,65 @@ +========================== +Messaging service overview +========================== + +The Message service is multi-tenant, fast, reliable, and scalable. It allows +developers to share data between distributed application components performing +different tasks, without losing messages or requiring each component to be +always available. + +The service features a RESTful API and a Websocket API, which developers can +use to send messages between various components of their SaaS and mobile +applications, by using a variety of communication patterns. + +Key features +~~~~~~~~~~~~ + +The Messaging service provides the following key features: + +* Choice between two communication transports. Both with Identity service + support: + + * Firewall-friendly, **HTTP-based RESTful API**. Many of today's developers + prefer a more web-friendly HTTP API. They value the simplicity and + transparency of the protocol, its firewall-friendly nature, and its huge + ecosystem of tools, load balancers and proxies. In addition, cloud + operators appreciate the scalability aspects of the REST architectural + style. + * **Websocket-based API** for persistent connections. Websocket protocol + provides communication over persistent connections. Unlike HTTP, where + new connections are opened for each request/response pair, Websocket can + transfer multiple requests/responses over single TCP connection. It saves + much network traffic and minimizes delays. + +* Multi-tenant queues based on Identity service IDs. +* Support for several common patterns including event broadcasting, task + distribution, and point-to-point messaging. +* Component-based architecture with support for custom back ends and message + filters. +* Efficient reference implementation with an eye toward low latency and high + throughput (dependent on back end). +* Highly-available and horizontally scalable. +* Support for subscriptions to queues. Several notification types are + available: + + * Email notifications + * Webhook notifications + * Websocket notifications + +Layers of the Messaging service +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The Messaging service has following layers: + +* The transport layer (Messaging application) which can provide these APIs: + + * HTTP RESTful API (via ``wsgi`` driver). + * Websocket API (via ``websocket`` driver). + +* The storage layer which keeps all the data and metadata about queues and + messages. It has two sub-layers: + + * The management store database (Catalog). Can be ``MongoDB`` database (or + ``MongoDB`` replica-set) or SQL database. + * The message store databases (Pools). Can be ``MongoDB`` database (or + ``MongoDB`` replica-set) or ``Redis`` database. diff --git a/install-guide/source/index.rst b/install-guide/source/index.rst new file mode 100644 index 000000000..a00da0061 --- /dev/null +++ b/install-guide/source/index.rst @@ -0,0 +1,22 @@ +================= +Messaging service +================= + +.. toctree:: + + get_started.rst + install.rst + verify.rst + next-steps.rst + +The Messaging service is multi-tenant, fast, reliable, and scalable. It allows +developers to share data between distributed application components performing +different tasks, without losing messages or requiring each component to be +always available. + +The service features a RESTful API and a Websocket API, which developers can +use to send messages between various components of their SaaS and mobile +applications, by using a variety of communication patterns. + +This chapter assumes a working setup of OpenStack following the base +Installation Guide. diff --git a/install-guide/source/install-rdo.rst b/install-guide/source/install-rdo.rst new file mode 100644 index 000000000..9f46b6320 --- /dev/null +++ b/install-guide/source/install-rdo.rst @@ -0,0 +1,545 @@ +.. _install-rdo: + +Install and configure for Red Hat Enterprise Linux and CentOS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This section describes how to install and configure the Messaging service, +code-named ``zaqar`` for Red Hat Enterprise Linux 7 and CentOS 7. + +This section assumes that you already have a working OpenStack environment with +at least Identity service installed. + +Here you can find instructions and recommended settings for installing +Messaging service in small configuration: one web server with Messaging service +configured to use replica-set of three ``MongoDB`` database servers. Because +only one web server is used, the Messaging service installed by using these +instructions can't be considered as high available, see :doc:`install`. + +In this tutorial these server names are used as examples: + +* Web server with Messaging service: ``WEB0.EXAMPLE-MESSAGES.NET``. +* Database servers: ``MYDB0.EXAMPLE-MESSAGES.NET``, + ``MYDB1.EXAMPLE-MESSAGES.NET``, ``MYDB2.EXAMPLE-MESSAGES.NET``. +* Identity service server: ``IDENTITY.EXAMPLE-MESSAGES.NET``. + +Prerequisites +------------- + +Before you install Messaging service, you must meet the following system +requirements: + +* Installed Identity service for user and project management. +* Python 2.7. + +Before you install and configure Messaging, you must create a ``MongoDB`` +replica-set of three database servers. Also you need to create service +credentials and API endpoints in Identity. + +#. Install and configure ``MongoDB`` replica-set on database servers: + + #. Install ``MongoDB`` on the database servers: + + On each database server follow the official `MongoDB installation + instructions`_. + + .. note:: + + Messaging service works with ``MongoDB`` versions >= 2.4 + + #. Configure ``MongoDB`` on the database servers: + + On each database server edit configuration file: ``/etc/mongod.conf`` and + modify as needed: + + .. code-block:: ini + + # MongoDB sample configuration for Messaging service. + # (For MongoDB version >= 2.6) + # Edit according to your needs. + systemLog: + destination: file + logAppend: true + path: /var/log/mongodb/mongod.log + + storage: + dbPath: /var/lib/mongo + journal: + enabled: false + + processManagement: + fork: true # fork and run in background + pidFilePath: /var/run/mongodb/mongod.pid # location of pidfile + + net: + port: 27017 + # bindIp: 127.0.0.1 # Listen to local interface only, comment to listen on all interfaces. + + operationProfiling: + slowOpThresholdMs: 200 + mode: slowOp + + replication: + oplogSizeMB: 2048 + replSetName: catalog + + .. note:: + + In case of older ``MongoDB`` versions (2.4 and 2.5) the configuration + file should be written in different format. For information about + format for different versions see the official `MongoDB configuration + reference`_. + + .. warning:: + + Additional steps are required to secure ``MongoDB`` installation. You + should modify this configuration for your security requirements. See + the official `MongoDB security reference`_. + + #. Start ``MongoDB`` on the database servers: + + Start ``MongoDB`` service on all database servers: + + .. code-block:: console + + root@MYDBX# systemctl start mongod + + Make ``MongoDB`` service start automatically after reboot: + + .. code-block:: console + + root@MYDBX# systemctl enable mongod + + #. Configure ``MongoDB`` Replica Set on the database servers: + + Once you've installed ``MongoDB`` on three servers and assuming that the + primary ``MongoDB`` server hostname is ``MYDB0.EXAMPLE-MESSAGES.NET``, go + to ``MYDB0`` and run these commands: + + .. code-block:: console + + root@MYDB0# mongo local --eval "printjson(rs.initiate())" + root@MYDB0# mongo local --eval "printjson(rs.add('MYDB1.EXAMPLE-MESSAGES.NET'))" + root@MYDB0# mongo local --eval "printjson(rs.add('MYDB2.EXAMPLE-MESSAGES.NET'))" + + .. note:: + + The database servers must have access to each other and also be + accessible from the Messaging service web server. Configure firewalls + on all database servers to accept incoming connections to port + ``27017`` from the needed source. + + To check if the replica-set is established see the output of this + command: + + .. code-block:: console + + root@MYDB0# mongo local --eval "printjson(rs.status())" + +#. Source the ``admin`` credentials to gain access to admin-only CLI commands: + + .. code-block:: console + + $ . admin-openrc + +#. To create the service credentials, complete these steps: + + #. Create the ``zaqar`` user: + + .. code-block:: console + + $ openstack user create --domain default --password-prompt zaqar + User Password: + Repeat User Password: + +-----------+----------------------------------+ + | Field | Value | + +-----------+----------------------------------+ + | domain_id | default | + | enabled | True | + | id | 7b0ffc83097148dab6ecbef6ddcc46bf | + | name | zaqar | + +-----------+----------------------------------+ + + #. Add the ``admin`` role to the ``zaqar`` user: + + .. code-block:: console + + $ openstack role add --project service --user zaqar admin + + .. note:: + + This command provides no output. + + #. Create the ``zaqar`` service entity: + + .. code-block:: console + + $ openstack service create --name zaqar --description "Messaging" messaging + +-------------+----------------------------------+ + | Field | Value | + +-------------+----------------------------------+ + | description | Messaging | + | enabled | True | + | id | b39c22818be5425ba2315dd4b10cd57c | + | name | zaqar | + | type | messaging | + +-------------+----------------------------------+ + +#. Create the Messaging service API endpoints: + + .. code-block:: console + + $ openstack endpoint create --region RegionOne messaging public http://WEB0.EXAMPLE-MESSAGES.NET:8888 + +--------------+---------------------------------------+ + | Field | Value | + +--------------+---------------------------------------+ + | enabled | True | + | id | aabca78860e74c4db0bcb36167bfe106 | + | interface | public | + | region | RegionOne | + | region_id | RegionOne | + | service_id | b39c22818be5425ba2315dd4b10cd57c | + | service_name | zaqar | + | service_type | messaging | + | url | http://WEB0.EXAMPLE-MESSAGES.NET:8888 | + +--------------+---------------------------------------+ + + $ openstack endpoint create --region RegionOne messaging internal http://WEB0.EXAMPLE-MESSAGES.NET:8888 + +--------------+---------------------------------------+ + | Field | Value | + +--------------+---------------------------------------+ + | enabled | True | + | id | 07f9524613de4fd3905e13a87f81fd3f | + | interface | internal | + | region | RegionOne | + | region_id | RegionOne | + | service_id | b39c22818be5425ba2315dd4b10cd57c | + | service_name | zaqar | + | service_type | messaging | + | url | http://WEB0.EXAMPLE-MESSAGES.NET:8888 | + +--------------+---------------------------------------+ + + $ openstack endpoint create --region RegionOne messaging admin http://WEB0.EXAMPLE-MESSAGES.NET:8888 + +--------------+---------------------------------------+ + | Field | Value | + +--------------+---------------------------------------+ + | enabled | True | + | id | 686f7b19428f4b5aa1425667dfe4f49d | + | interface | admin | + | region | RegionOne | + | region_id | RegionOne | + | service_id | b39c22818be5425ba2315dd4b10cd57c | + | service_name | zaqar | + | service_type | messaging | + | url | http://WEB0.EXAMPLE-MESSAGES.NET:8888 | + +--------------+---------------------------------------+ + +Install and configure Messaging web server +------------------------------------------ + +Install and configure ``memcached``, ``uWSGI`` and Messaging on the web server +``WEB0.EXAMPLE-MESSAGES.NET``. + +#. Install ``memcached`` on web server ``WEB0.EXAMPLE-MESSAGES.NET`` in order + to cache Identity service tokens and catalog mappings: + + .. code-block:: console + + root@WEB0# yum install memcached + + Start ``memcached`` service: + + .. code-block:: console + + root@WEB0# systemctl start memcached + + Make ``memcached`` service start automatically after reboot: + + .. code-block:: console + + root@WEB0# systemctl enable memcached + +#. Install Messaging service and ``uWSGI``: + + .. code-block:: console + + root@WEB0# yum -y install python-pip + root@WEB0# git clone https://git.openstack.org/openstack/zaqar.git + root@WEB0# cd zaqar + root@WEB0# pip install . -r ./requirements.txt --upgrade --log /tmp/zaqar-pip.log + root@WEB0# pip install --upgrade pymongo gevent uwsgi + +#. Copy the Zaqar RBAC policy sample file to the directory ``etc/zaqar/``: + + .. code-block:: console + + root@WEB0# mkdir + root@WEB0# cp etc/policy.json.sample /etc/zaqar/policy.json + +#. Create log file: + + .. code-block:: console + + root@WEB0# touch /var/log/zaqar-server.log + root@WEB0# chown ZAQARUSER:ZAQARUSER /var/log/zaqar-server.log + root@WEB0# chmod 600 /var/log/zaqar-server.log + + Replace ``ZAQARUSER`` with the name of the user in system under which the + Messaging service will run. + +#. Create ``/srv/zaqar`` folder to store ``uWSGI`` configuration files. + +#. Create ``/srv/zaqar/zaqar_uwsgi.py`` with the following content: + + .. code-block:: python + + from keystonemiddleware import auth_token + from zaqar.transport.wsgi import app + + app = auth_token.AuthProtocol(app.app, {}) + +#. Increase backlog listen limit from default (128): + + .. code-block:: console + + root@WEB0# echo "net.core.somaxconn=2048" | sudo tee --append /etc/sysctl.conf + +#. Create ``/srv/zaqar/uwsgi.ini`` file with the following content and modify + as needed: + + .. code-block:: ini + + [uwsgi] + https = WEB0.EXAMPLE-MESSAGES.NET:8888,PATH_TO_SERVER_CRT,PATH_TO_SERVER_PRIVATE_KEY + pidfile = /var/run/zaqar.pid + gevent = 2000 + gevent-monkey-patch = true + listen = 1024 + enable-threads = true + module = zaqar_uwsgi:app + workers = 4 + harakiri = 60 + add-header = Connection: close + + Replace ``PATH_TO_SERVER_CRT`` with path to the server's certificate + (``*.crt``) and ``PATH_TO_SERVER_PRIVATE_KEY`` with path to the server's + private key (``*.key``). + + .. note:: + + The ``uWSGI`` configuration options above can be modified for different + security and performance requirements including load balancing. See the + official `uWSGI configuration reference`_. + +#. Create Messaging service's configuration file ``/etc/zaqar.conf`` with the + following content: + + .. code-block:: ini + + [DEFAULT] + # Show debugging output in logs (sets DEBUG log level output) + #debug = False + + # Pooling and admin mode configs + pooling = True + admin_mode = True + + # Log to file + log_file = /var/log/zaqar-server.log + + # This is taken care of in our custom app.py, so disable here + ;auth_strategy = keystone + + # Modify to make it work with your Identity service. + [keystone_authtoken] + project_domain_name = Default + user_domain_name = Default + project_domain_id = default + project_name = service + user_domain_id = default + # File path to a PEM encoded Certificate Authority to use when verifying + # HTTPs connections. Defaults to system CAs if commented. + cafile = PATH_TO_CA_FILE + # Messaging service user name in Identity service. + username = ZAQARIDENTITYUSER + # Messaging service password in Identity service. + password = ZAQARIDENTITYPASSWORD + # Complete public Identity API endpoint (HTTPS protocol is more preferable + # than HTTP). + auth_uri = HTTPS://IDENTITY.EXAMPLE-MESSAGES.NET:5000 + # Complete admin Identity API endpoint (HTTPS protocol is more preferable + # than HTTP). + auth_url = HTTPS://IDENTITY.EXAMPLE-MESSAGES.NET:35357 + # Token cache time in seconds. + token_cache_time = TOKEN_CACHE_TIME + memcached_servers = 127.0.0.1:11211 + + [cache] + # Dogpile.cache backend module. It is recommended that Memcache with + # pooling (oslo_cache.memcache_pool) or Redis (dogpile.cache.redis) be + # used in production deployments. Small workloads (single process) + # like devstack can use the dogpile.cache.memory backend. (string + # value) + backend = dogpile.cache.memory + memcache_servers = 127.0.0.1:11211 + + [drivers] + transport = wsgi + message_store = mongodb + management_store = mongodb + + [drivers:management_store:mongodb] + # Mongodb Connection URI. If ssl connection enabled, then ssl_keyfile, + # ssl_certfile, ssl_cert_reqs, ssl_ca_certs options need to be set + # accordingly. + uri = mongodb://MYDB0.EXAMPLE-MESSAGES.NET,MYDB1.EXAMPLE-MESSAGES.NET,MYDB2.EXAMPLE-MESSAGES.NET:27017/?replicaSet=catalog&w=2&readPreference=secondaryPreferred + + # Name for the database on mongodb server. + database = zaqarmanagementstore + + # Number of databases across which to partition message data, in order + # to reduce writer lock %. DO NOT change this setting after initial + # deployment. It MUST remain static. Also, you should not need a large + # number of partitions to improve performance, esp. if deploying + # MongoDB on SSD storage. (integer value) + partitions = 8 + + # Uncomment any options below if needed. + + # Maximum number of times to retry a failed operation. Currently + # only used for retrying a message post. + ;max_attempts = 1000 + + # Maximum sleep interval between retries (actual sleep time + # increases linearly according to number of attempts performed). + ;max_retry_sleep = 0.1 + + # Maximum jitter interval, to be added to the sleep interval, in + # order to decrease probability that parallel requests will retry + # at the same instant. + ;max_retry_jitter = 0.005 + + # Frequency of message garbage collections, in seconds + ;gc_interval = 5 * 60 + + # Threshold of number of expired messages to reach in a given + # queue, before performing the GC. Useful for reducing frequent + # locks on the DB for non-busy queues, or for worker queues + # which process jobs quickly enough to keep the number of in- + # flight messages low. + # + # Note: The higher this number, the larger the memory-mapped DB + # files will be. + ;gc_threshold = 1000 + + [drivers:message_store:mongodb] + # This section has same set of available options as + # "[drivers:management_store:mongodb]" section. + # + # If pooling is enabled, all pools inherit values from options in these + # settings unless overridden in pool creation request. Also "uri" option + # value isn't used in case of pooling. + # + # If ssl connection enabled, then ssl_keyfile, ssl_certfile, ssl_cert_reqs, + # ssl_ca_certs options need to be set accordingly. + + # Name for the database on MondoDB server. + database = zaqarmessagestore + + [transport] + max_queues_per_page = 1000 + max_queue_metadata = 262144 + max_mesages_per_page = 10 + max_messages_post_size = 262144 + max_message_ttl = 1209600 + max_claim_ttl = 43200 + max_claim_grace = 43200 + + [signed_url] + # Secret key used to encrypt pre-signed URLs. (string value) + secret_key = SOMELONGSECRETKEY + + Edit any options as needed, especially the options with capitalized values. + +#. Create a service file for Messaging service + ``/etc/systemd/system/zaqaruwsgi.service``: + + .. code-block:: ini + + [Unit] + Description=uWSGI Zaqar + After=syslog.target + + [Service] + ExecStart=/usr/bin/uwsgi --ini /srv/zaqar/uwsgi.ini + # Requires systemd version 211 or newer + RuntimeDirectory=uwsgi + Restart=always + KillSignal=SIGQUIT + Type=notify + StandardError=syslog + NotifyAccess=all + User=ZAQARUSER + Group=ZAQARUSER + + [Install] + WantedBy=multi-user.target + + Replace ``ZAQARUSER`` with the name of the user in system under which the + Messaging service will run. + +Finalize installation +--------------------- + +Now after you have configured the web server and the database servers to have a +functional Messaging service, you need to start the service, make the service +automatically start with the system and define the created ``MongoDB`` +replica-set as Messaging's pool. + +#. Start Messaging service on the web server: + + .. code-block:: console + + root@WEB0# systemctl start zaqar.uwsgi.service + +#. Make Messaging service start automatically after reboot on the web server: + + .. code-block:: console + + root@WEB0# systemctl enable zaqar.uwsgi.service + +#. Configure pool: + + .. code-block:: console + + root@WEB0# curl -i -X PUT https://WEB0.EXAMPLE-MESSAGES.NET:8888/v2/pools/POOL1 \ + -d '{"weight": 100, "uri": "mongodb://MYDB0.EXAMPLE-MESSAGES.NET,MYDB1.EXAMPLE-MESSAGES.NET,MYDB2.EXAMPLE-MESSAGES.NET:27017/?replicaSet=catalog&w=2&readPreference=secondaryPreferred", "options": {"partitions": 8}}' \ + -H "Client-ID: CLIENT_ID" \ + -H "X-Auth-Token: TOKEN" \ + -H "Content-type: application/json" \ + + Replace ``POOL1`` variable with the desired name of a pool. + + Replace ``CLIENT_ID`` variable with the universally unique identifier (UUID) + which can be generated by, for example, ``uuidgen`` utility. + + Replace ``TOKEN`` variable with the authentication token retrieved from + Identity service. If you choose not to enable Keystone authentication you + won't have to pass a token. + + .. note:: + + The ``options`` key in curl request above overrides any options + (specified in configuration file or default) in + ``[drivers:message_store:mongodb]`` Messaging service configuration + file's section. + +.. tip:: + + In larger deployments, there should be many load balanced web servers. Also + the management store databases and the message store databases (pools) + should be on different ``MongoDB`` replica-sets. + +.. _`MongoDB installation instructions`: https://docs.mongodb.org/manual/tutorial/install-mongodb-on-red-hat/ +.. _`MongoDB configuration reference`: https://docs.mongodb.org/v3.0/reference/configuration-options/ +.. _`MongoDB security reference`: https://docs.mongodb.org/manual/security/ +.. _`uWSGI configuration reference`: http://uwsgi-docs.readthedocs.io/en/latest/ diff --git a/install-guide/source/install.rst b/install-guide/source/install.rst new file mode 100644 index 000000000..7ae2a593c --- /dev/null +++ b/install-guide/source/install.rst @@ -0,0 +1,33 @@ +.. _install: + +Install and configure +~~~~~~~~~~~~~~~~~~~~~ + +This section describes how to install and configure the Messaging service, +code-named zaqar. + +This section assumes that you already have a working OpenStack environment with +at least Identity service installed. + +Note that installation and configuration vary by distribution. + +.. toctree:: + + install-rdo.rst + +Possible Minimum Scalable HA Setup +---------------------------------- + +Scalable HA (High availability) setup is out of scope in this chapter. + +For a HA setup, a load balancer has to be placed in front of the web servers. + +To provide high availability with minimum administration overhead for storage +use ``MongoDB`` driver and for transport use ``wsgi`` driver. + +To have a small footprint while providing HA, you can use two web servers which +will host the application and three ``MongoDB`` servers (configured as +replica-set) which will host Messaging service's management store and +message store databases. At larger scale, the management store database and the +message store database are advised to be hosted on different ``MongoDB`` +replica sets. diff --git a/install-guide/source/next-steps.rst b/install-guide/source/next-steps.rst new file mode 100644 index 000000000..2c4f455de --- /dev/null +++ b/install-guide/source/next-steps.rst @@ -0,0 +1,9 @@ +.. _next-steps: + +Next steps +~~~~~~~~~~ + +Your OpenStack environment now includes the Messaging service. + +To add additional services, see the +`additional documentation on installing OpenStack `_ . diff --git a/install-guide/source/verify.rst b/install-guide/source/verify.rst new file mode 100644 index 000000000..536591d9c --- /dev/null +++ b/install-guide/source/verify.rst @@ -0,0 +1,36 @@ +.. _verify: + +Verify operation +~~~~~~~~~~~~~~~~ + +Verify operation of the Messaging service by creating messages via curl +utility: + +.. code-block:: console + + $ curl -i -X POST http://ZAQAR_ENDPOINT:8888/v2/queues/samplequeue/messages \ + -d '{"messages": [{"body": {"event": 1}, "ttl": 600}, {"body": {"event": 2}, "ttl": 600}]}' \ + -H "Content-type: application/json" \ + -H "Client-ID: CLIENT_ID" \ + -H "X-Auth-ToKen: TOKEN" + +Replace ``CLIENT_ID`` variable with the universally unique identifier (UUID) +which can be generated by, for example, ``uuidgen`` utility. + +Replace ``TOKEN`` variable with the authentication token retrieved from +Identity service. If you choose not to enable Keystone authentication you +won't have to pass a token. + +Replace ``ZAQAR_ENDPOINT`` variable with the endpoint of Messaging service. + +The normal response would be with status code 201 and look something like this: + +.. code-block:: console + + HTTP/1.1 201 Created + content-length: 135 + content-type: application/json; charset=UTF-8 + location: http://ZAQAR_ENDPOINT:8888/v2/queues/samplequeue/messages?ids=575f6f2515e5c87d779a9b20,575f6f2515e5c87d779a9b21 + Connection: close + + {"resources": ["/v2/queues/samplequeue/messages/575f6f2515e5c87d779a9b20", "/v2/queues/samplequeue/messages/575f6f2515e5c87d779a9b21"]} diff --git a/test-requirements.txt b/test-requirements.txt index 3b8fb765e..a03d71264 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -24,6 +24,7 @@ testtools>=1.4.0 # MIT sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0 openstack-doc-tools>=0.23 # Apache-2.0 +openstackdocstheme>=1.0.3 # Apache-2.0 oslotest>=1.10.0 # Apache-2.0 reno>=1.6.2 # Apache2 os-api-ref>=0.1.0 # Apache-2.0 diff --git a/tox.ini b/tox.ini index dbe0079c4..ccdff93ad 100644 --- a/tox.ini +++ b/tox.ini @@ -77,3 +77,9 @@ max-complexity = 16 [hacking] import_exceptions = zaqar.openstack.common.gettextutils._,zaqar.i18n._ +[testenv:install-guide] +# NOTE(jaegerandi): this target does not use constraints because +# upstream infra does not yet support it. Once that's fixed, we can +# drop the install_command. +install_command = pip install -U --force-reinstall {opts} {packages} +commands = sphinx-build -a -E -W -d install-guide/build/doctrees -b html install-guide/source install-guide/build/html \ No newline at end of file