Documentation update

- added documentation for kafka configuration
- moved python documentation to own part
 - added section for coverage and building

Change-Id: I0998c955ee652bbe32af152b9c1ae894f06c1396
This commit is contained in:
Tomasz Trębski 2015-11-18 09:28:35 +01:00
parent dc89cde325
commit 59f58c7553
20 changed files with 1062 additions and 50 deletions

192
Makefile Normal file
View File

@ -0,0 +1,192 @@
# Makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
PAPER =
BUILDDIR = build
# User-friendly check for sphinx-build
ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
endif
# Internal variables.
PAPEROPT_a4 = -D latex_paper_size=a4
PAPEROPT_letter = -D latex_paper_size=letter
ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) documentation
# the i18n builder cannot share the environment and doctrees with the others
I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) documentation
.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext
help:
@echo "Please use \`make <target>' where <target> is one of"
@echo " html to make standalone HTML files"
@echo " dirhtml to make HTML files named index.html in directories"
@echo " singlehtml to make a single large HTML file"
@echo " pickle to make pickle files"
@echo " json to make JSON files"
@echo " htmlhelp to make HTML files and a HTML help project"
@echo " qthelp to make HTML files and a qthelp project"
@echo " applehelp to make an Apple Help Book"
@echo " devhelp to make HTML files and a Devhelp project"
@echo " epub to make an epub"
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
@echo " latexpdf to make LaTeX files and run them through pdflatex"
@echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
@echo " text to make text files"
@echo " man to make manual pages"
@echo " texinfo to make Texinfo files"
@echo " info to make Texinfo files and run them through makeinfo"
@echo " gettext to make PO message catalogs"
@echo " changes to make an overview of all changed/added/deprecated items"
@echo " xml to make Docutils-native XML files"
@echo " pseudoxml to make pseudoxml-XML files for display purposes"
@echo " linkcheck to check all external links for integrity"
@echo " doctest to run all doctests embedded in the documentation (if enabled)"
@echo " coverage to run coverage check of the documentation (if enabled)"
clean:
rm -rf $(BUILDDIR)/*
html:
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
dirhtml:
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
singlehtml:
$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
@echo
@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
pickle:
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
@echo
@echo "Build finished; now you can process the pickle files."
json:
$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
@echo
@echo "Build finished; now you can process the JSON files."
htmlhelp:
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
@echo
@echo "Build finished; now you can run HTML Help Workshop with the" \
".hhp project file in $(BUILDDIR)/htmlhelp."
qthelp:
$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
@echo
@echo "Build finished; now you can run "qcollectiongenerator" with the" \
".qhcp project file in $(BUILDDIR)/qthelp, like this:"
@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/monasca-log-api.qhcp"
@echo "To view the help file:"
@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/monasca-log-api.qhc"
applehelp:
$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
@echo
@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
@echo "N.B. You won't be able to view it unless you put it in" \
"~/Library/Documentation/Help or install it in your application" \
"bundle."
devhelp:
$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
@echo
@echo "Build finished."
@echo "To view the help file:"
@echo "# mkdir -p $$HOME/.local/share/devhelp/monasca-log-api"
@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/monasca-log-api"
@echo "# devhelp"
epub:
$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
@echo
@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
latex:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo
@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
@echo "Run \`make' in that directory to run these through (pdf)latex" \
"(use \`make latexpdf' here to do that automatically)."
latexpdf:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through pdflatex..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
latexpdfja:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through platex and dvipdfmx..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
text:
$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
@echo
@echo "Build finished. The text files are in $(BUILDDIR)/text."
man:
$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
@echo
@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
texinfo:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo
@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
@echo "Run \`make' in that directory to run these through makeinfo" \
"(use \`make info' here to do that automatically)."
info:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo "Running Texinfo files through makeinfo..."
make -C $(BUILDDIR)/texinfo info
@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
gettext:
$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
@echo
@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
changes:
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
@echo
@echo "The overview file is in $(BUILDDIR)/changes."
linkcheck:
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
@echo
@echo "Link check complete; look for any errors in the above output " \
"or in $(BUILDDIR)/linkcheck/output.txt."
doctest:
$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
@echo "Testing of doctests in the sources finished, look at the " \
"results in $(BUILDDIR)/doctest/output.txt."
coverage:
$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
@echo "Testing of coverage in the sources finished, look at the " \
"results in $(BUILDDIR)/coverage/python.txt."
xml:
$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
@echo
@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
pseudoxml:
$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
@echo
@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."

View File

@ -76,54 +76,11 @@ Requests flow through the following architectural layers from top to bottom:
## Documentation
* API Specification: [/docs/monasca-log-api-spec.md](/docs/monasca-log-api-spec.md).
* Kafka communication: [/docs/monasca-log-api-kafka.md](/docs/monasca-log-api-kafka.md).
## Python monasca-log-api implementation
To install the python api implementation, git clone the source and run the
following command::
```sh
sudo python setup.py install
```
If it installs successfully, you will need to make changes to the following
two files to reflect your system settings, especially where kafka server is
located::
```sh
/etc/monasca/log-api.conf
/etc/monasca/log-api.ini
```
Once the configurations are modified to match your environment, you can start
up the server by following the following instructions.
To start the server, run the following command:
Running the server in foreground mode
```sh
gunicorn -k eventlet --worker-connections=2000 --backlog=1000
--paste /etc/monasca/log-api-config.ini
```
Running the server as daemons
```sh
gunicorn -k eventlet --worker-connections=2000 --backlog=1000
--paste /etc/monasca/log-api-config.ini -D
```
To check if the code follows python coding style, run the following command
from the root directory of this project
```sh
tox -e pep8
```
To run all the unit test cases, run the following command from the root
directory of this project
```sh
tox -e py27 (or -e py26, -e py33)
```
See here [/monasca_log_api/README.md](/monasca_log_api/README.md).
# License

View File

@ -0,0 +1,106 @@
# Monasca Log API - Kafka
Date: November 18, 2015
Document Version: v0.1
## Introduction
**monasca-log-api** uses kafka transport to ship received logs down to the
processing pipeline.
For more information about Kafka, please see [official documentation]
(http://kafka.apache.org/documentation.html).
## Output message format
Messages sent to kafka should have following format
(top level object is called **envelope**) and is combined out of three
elements:
* log
* creation_time
* meta
"log": <object>,
"creation_time": <number, timestamp>,
"meta": <object>
Log property should have at least following form:
"message": <string>,
"application_type": <string>,
"dimensions": <object>
Meta property should have following form:
"tenantId": <string>,
"region": <string>
Full example as json:
```json
{
"log": {
"message": "2015-11-13 12:44:42.411 27297 DEBUG kafka [-] Read 31/31 bytes from Kafka _read_bytes /opt/monasca/monasca-log-api/lib/python2.7/site-packages/kafka/conn.py:103",
"application_type": "monasca-log-api",
"dimension": {
"hostname": "devstack"
}
},
"creation_time": 1447834886,
"meta": {
"tenantId": "e4bd29509eda473092d32aadfee3e7b1",
"region": "pl"
}
}
```
### Fields explanation
* log - contains log specific information collected from the system. In the
most lean case that would be: **message**, **application_type**, **dimensions**
* message - normally that represent a single line from a log file
* application_type - represent the application that log was collected
* dimensions - informations such as hostname where application is running
* creation_time - UNIX timestamp representing moment when log message was created
by monasca-log-api
* meta - contains tenantId and its region
**log** entry main of course contain many more fields that are considered valid
in given case. However three mentioned in this documentation are required.
All fields, apart from **creation_time** and **log**, are created from HTTP headers.
Description is available [here](/docs/monasca-log-api-spec.md).
## Configuration
### Java
Configuration for kafka should be placed in *.yml file and look similar to:
```yml
logTopic: logs
kafka:
brokerUris:
- localhost:8900
zookeeperUris:
- localhost:2181
healthCheckTopic: healthcheck
```
It is composed out of two relevant pieces
* logTopic - topic where data should be sent
* kafka - section containing information required to communicate in kafka.
For more details see [here](https://github.com/openstack/monasca-common/blob/master/java/monasca-common-kafka/src/main/java/monasca/common/messaging/kafka/KafkaConfiguration.java)
### Python
Configuration for kafka should be placed in *.conf file and look similar to:
```conf
[log_publisher]
topics = 'logs'
kafka_url = 'localhost:8900'
```
There are only two relevant options:
* topics - comma delimited list of topics where data should be sent
* kafka_url - adress where kafka server is running

View File

@ -1,8 +1,8 @@
# Monasca Log API
Date: August 19, 2015
Date: November 18, 2015
Document Version: v2.0
Document Version: v2.1
# Log
The log resource allows logs to be created.

304
documentation/conf.py Normal file
View File

@ -0,0 +1,304 @@
# -*- coding: utf-8 -*-
#
# monasca-log-api documentation build configuration file, created by
# sphinx-quickstart on Wed Nov 18 12:02:03 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
import sys
import os
import shlex
import sphinx_rtd_theme
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))
# -- General configuration ------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.doctest',
'sphinx.ext.intersphinx',
'sphinx.ext.todo',
'sphinx.ext.coverage',
'sphinx.ext.viewcode',
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'
# The encoding of source files.
#source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = u'monasca-log-api'
copyright = u'2015, FUJITSU LIMITED'
author = u'Tomasz Trębski <tomasz.trebski@ts.fujitsu.com>,Lukasz Zajaczkowski <lukasz.zajaczkowski@ts.fujitsu.com>'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.0.1'
# The full version, including alpha/beta/rc tags.
release = '0.0.1'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = [
'common',
'docs',
'documentation',
'etc',
'java'
]
# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
show_authors = True
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []
# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False
# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True
# -- Options for HTML output ----------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = 'sphinx_rtd_theme'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
html_theme_path = [
sphinx_rtd_theme.get_html_theme_path()
]
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
#html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
# If false, no module index is generated.
#html_domain_indices = True
# If false, no index is generated.
#html_use_index = True
# If true, the index is split into individual pages for each letter.
#html_split_index = False
# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''
# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None
# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'
# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}
# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'
# Output file base name for HTML help builder.
htmlhelp_basename = 'monasca-log-apidoc'
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#'preamble': '',
# Latex figure (float) alignment
#'figure_align': 'htbp',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'monasca-log-api.tex', u'monasca-log-api Documentation',
u'Tomasz Trębski \\textless{}tomasz.trebski@ts.fujitsu.com\\textgreater{}', 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False
# If true, show page references after internal links.
#latex_show_pagerefs = False
# If true, show URL addresses after external links.
#latex_show_urls = False
# Documents to append as an appendix to all manuals.
#latex_appendices = []
# If false, no module index is generated.
#latex_domain_indices = True
# -- Options for manual page output ---------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'monasca-log-api', u'monasca-log-api Documentation',
[author], 1)
]
# If true, show URL addresses after external links.
#man_show_urls = False
# -- Options for Texinfo output -------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'monasca-log-api', u'monasca-log-api Documentation',
author, 'monasca-log-api', 'One line description of project.',
'Miscellaneous'),
]
# Documents to append as an appendix to all manuals.
#texinfo_appendices = []
# If false, no module index is generated.
#texinfo_domain_indices = True
# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'
# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False
# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'https://docs.python.org/': None}

22
documentation/index.rst Normal file
View File

@ -0,0 +1,22 @@
.. monasca-log-api documentation master file, created by
sphinx-quickstart on Wed Nov 18 12:02:03 2015.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Welcome to monasca-log-api's documentation!
===========================================
Contents:
.. toctree::
:maxdepth: 2
monasca_log_api
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

View File

@ -0,0 +1,54 @@
monasca_log_api.api package
===========================
Submodules
----------
monasca_log_api.api.exceptions module
-------------------------------------
.. automodule:: monasca_log_api.api.exceptions
:members:
:undoc-members:
:show-inheritance:
monasca_log_api.api.headers module
----------------------------------
.. automodule:: monasca_log_api.api.headers
:members:
:undoc-members:
:show-inheritance:
monasca_log_api.api.logs_api module
-----------------------------------
.. automodule:: monasca_log_api.api.logs_api
:members:
:undoc-members:
:show-inheritance:
monasca_log_api.api.rest_utils module
-------------------------------------
.. automodule:: monasca_log_api.api.rest_utils
:members:
:undoc-members:
:show-inheritance:
monasca_log_api.api.versions_api module
---------------------------------------
.. automodule:: monasca_log_api.api.versions_api
:members:
:undoc-members:
:show-inheritance:
Module contents
---------------
.. automodule:: monasca_log_api.api
:members:
:undoc-members:
:show-inheritance:

View File

@ -0,0 +1,30 @@
monasca_log_api package
=======================
Subpackages
-----------
.. toctree::
monasca_log_api.api
monasca_log_api.v2
Submodules
----------
monasca_log_api.server module
-----------------------------
.. automodule:: monasca_log_api.server
:members:
:undoc-members:
:show-inheritance:
Module contents
---------------
.. automodule:: monasca_log_api
:members:
:undoc-members:
:show-inheritance:

View File

@ -0,0 +1,30 @@
monasca_log_api.v2.common package
=================================
Submodules
----------
monasca_log_api.v2.common.log_publisher module
----------------------------------------------
.. automodule:: monasca_log_api.v2.common.log_publisher
:members:
:undoc-members:
:show-inheritance:
monasca_log_api.v2.common.service module
----------------------------------------
.. automodule:: monasca_log_api.v2.common.service
:members:
:undoc-members:
:show-inheritance:
Module contents
---------------
.. automodule:: monasca_log_api.v2.common
:members:
:undoc-members:
:show-inheritance:

View File

@ -0,0 +1,30 @@
monasca_log_api.v2.reference package
====================================
Submodules
----------
monasca_log_api.v2.reference.logs module
----------------------------------------
.. automodule:: monasca_log_api.v2.reference.logs
:members:
:undoc-members:
:show-inheritance:
monasca_log_api.v2.reference.versions module
--------------------------------------------
.. automodule:: monasca_log_api.v2.reference.versions
:members:
:undoc-members:
:show-inheritance:
Module contents
---------------
.. automodule:: monasca_log_api.v2.reference
:members:
:undoc-members:
:show-inheritance:

View File

@ -0,0 +1,18 @@
monasca_log_api.v2 package
==========================
Subpackages
-----------
.. toctree::
monasca_log_api.v2.common
monasca_log_api.v2.reference
Module contents
---------------
.. automodule:: monasca_log_api.v2
:members:
:undoc-members:
:show-inheritance:

82
monasca_log_api/README.md Normal file
View File

@ -0,0 +1,82 @@
# Monasca-log-api Python
## Installation
To install the python api implementation, git clone the source and run the
following command::
```sh
sudo python setup.py install
```
If it installs successfully, you will need to make changes to the following
two files to reflect your system settings, especially where kafka server is
located::
```sh
/etc/monasca/log-api-config.conf
/etc/monasca/log-api-config.ini
```
Once the configurations are modified to match your environment, you can start
up the server by following the following instructions.
To start the server, run the following command:
Running the server in foreground mode
```sh
gunicorn -k eventlet --worker-connections=2000 --backlog=1000
--paste /etc/monasca/log-api.ini
```
Running the server as daemons
```sh
gunicorn -k eventlet --worker-connections=2000 --backlog=1000
--paste /etc/monasca/log-api.ini -D
```
## Testing
### PEP8 guidelines
To check if the code follows python coding style, run the following command
from the root directory of this project:
```sh
tox -e pep8
```
### Testing
To run all the unit test cases, run the following command from the root
directory of this project:
```sh
tox -e py27 (or -e py26, -e py33)
```
### Coverage
To generate coverage results, run the following command from the root
directory of this project:
```sh
tox -e cover
```
### Building
To build an installable package, run the following command from the root
directory of this project:
```sh
python setup.py sdist
```
### Documentation
To generate documentation, run the following command from the root
directory of this project:
```sh
make html
```
That will create documentation under build folder relative to root of the
project.

View File

@ -20,12 +20,14 @@ HTTP_422 = '422 Unprocessable Entity'
class HTTPUnprocessableEntity(falcon.OptionalRepresentation, falcon.HTTPError):
"""HTTPUnprocessableEntity http error
"""HTTPUnprocessableEntity http error.
HTTPError that comes with '422 Unprocessable Entity' status
Args:
message(str) - meaningful description of what caused an error
:argument: message(str) - meaningful description of what caused an error
:argument: kwargs - any other option defined in
:py:class:`falcon.OptionalRepresentation` and
:py:class:`falcon.HTTPError`
"""
def __init__(self, message, **kwargs):
falcon.HTTPError.__init__(self,

View File

@ -16,6 +16,7 @@
import collections
LogApiHeader = collections.namedtuple('LogApiHeader', ['name', 'is_required'])
"""Tuple describing a header."""
X_TENANT_ID = LogApiHeader(name='X-Tenant-Id', is_required=False)
X_ROLES = LogApiHeader(name='X-Roles', is_required=False)

View File

@ -22,9 +22,25 @@ MONITORING_DELEGATE_ROLE = 'monitoring-delegate'
class LogsApi(object):
"""Logs API.
Logs API acts as RESTful endpoint accepting
messages contains collected log entries from the system.
Works as gateway for any further processing for accepted data.
"""
def __init__(self):
super(LogsApi, self).__init__()
LOG.info('Initializing LogsApi!')
def on_post(self, req, res):
"""Accepts sent logs as text or json.
Accepts logs sent to resource which should
be sent to kafka queue.
:param req: current request
:param res: current response
"""
res.status = falcon.HTTP_501

View File

@ -20,6 +20,23 @@ ENCODING = 'utf8'
def read_body(payload, content_type='application/json'):
"""Reads HTTP payload according to given content_type.
Function is capable of reading from payload stream.
Read data is then processed according to content_type.
Note:
There is no transformation if content type is equal to
'text/plain'. What has been read is returned.
:param payload(stream): payload to read
:param content_type(str): payload content type
:return: read data, returned type depends on content_type
:exception: :py:class:`falcon.HTTPBadRequest` - in case of any failure when
reading data
"""
try:
content = payload.read()
if not content:
@ -39,6 +56,11 @@ def read_body(payload, content_type='application/json'):
def as_json(data):
"""Writes data as json.
:param data(dict): data to convert to json
:return (str): json string
"""
return json.dumps(data,
encoding=ENCODING,
sort_keys=False,
@ -46,4 +68,9 @@ def as_json(data):
def from_json(data):
"""Reads data from json str.
:param data(str): data to read
:return (dict): read data
"""
return json.loads(data, encoding=ENCODING)

View File

@ -20,6 +20,12 @@ LOG = log.getLogger(__name__)
class VersionsAPI(object):
"""Versions API
VersionsAPI returns information about API itself.
"""
def __init__(self):
super(VersionsAPI, self).__init__()
LOG.info('Initializing VersionsAPI!')

View File

@ -43,6 +43,23 @@ class InvalidMessageException(Exception):
class LogPublisher(object):
"""Publishes log data to Kafka
LogPublisher is able to send single message to multiple configured topic.
It uses following configuration written in conf file ::
[log_publisher]
topics = 'logs'
kafka_url = 'localhost:8900'
Note:
Uses :py:class:`monasca_common.kafka.producer.KafkaProducer`
to ship logs to kafka. For more details
see `monasca_common`_ github repository.
.. _monasca_common: https://github.com/openstack/monasca-common
"""
def __init__(self):
self._topics = CONF.log_publisher.topics
self._kafka_publisher = None
@ -115,6 +132,23 @@ class LogPublisher(object):
return self._kafka_publisher
def send_message(self, message):
"""Sends message to each configured topic.
Prior to sending a message, unique key is being
calculated for the given message using:
* tenant id
* application type
* dimensions
Note:
Falsy messages (i.e. empty) are not shipped to kafka
See :py:meth:`monasca_log_api.v2.common.service.LogCreator`
`.new_log_envelope`
:param dict message: instance of log envelope
"""
if not message:
return
if not self._is_message_valid(message):

View File

@ -40,13 +40,25 @@ APPLICATION_TYPE_CONSTRAINTS = {
'MAX_LENGTH': 255,
'PATTERN': re.compile('^[a-zA-Z0-9_\\.\\-]+$')
}
"""Application type constraint used in validation.
See :py:func:`Validations.validate_application_type`
"""
DIMENSION_NAME_CONSTRAINTS = {
'MAX_LENGTH': 255,
'PATTERN': re.compile('[^><={}(), \'";&]+$')
}
"""Constraint for name of single dimension.
See :py:func:`Validations.validate_dimensions`
"""
DIMENSION_VALUE_CONSTRAINTS = {
'MAX_LENGTH': 255
}
"""Constraint for value of single dimension.
See :py:func:`Validations.validate_dimensions`
"""
EPOCH_START = datetime.datetime(1970, 1, 1)
@ -55,9 +67,15 @@ class LogEnvelopeException(Exception):
class Validations(object):
"""Contains validation logic."""
@staticmethod
def validate_application_type(application_type=None):
"""Validates application type.
Validation won't take place if application_type is None.
For details see: :py:data:`APPLICATION_TYPE_CONSTRAINTS`
"""
def validate_length():
if (len(application_type) >
@ -86,6 +104,14 @@ class Validations(object):
@staticmethod
def validate_dimensions(dimensions):
"""Validates dimensions type.
Empty dimensions are not being validated.
For details see:
* :py:data:`DIMENSION_NAME_CONSTRAINTS`
* :py:data:`DIMENSION_VALUE_CONSTRAINTS`
"""
def validate_name(name):
if not name:
@ -132,12 +158,36 @@ class Validations(object):
class LogCreator(object):
"""Transforms logs,
Takes care of transforming information received via
HTTP requests into log and log envelopes objects.
For more details see following:
* :py:func:`LogCreator.new_log`
* :py:func:`LogCreator.new_log_envelope`
"""
def __init__(self):
self._log = log.getLogger('service.LogCreator')
self._log.info('Initializing LogCreator')
@staticmethod
def _create_meta_info(tenant_id):
"""Creates meta block for log envelope.
Additionally method accesses oslo configuration,
looking for *service.region* configuration property.
For more details see :py:data:`service_opts`
:param tenant_id: ID of the tenant
:type tenant_id: str
:return: meta block
:rtype: dict
"""
return {
'tenantId': tenant_id,
'region': cfg.CONF.service.region
@ -149,6 +199,19 @@ class LogCreator(object):
payload,
content_type='application/json',
validate=True):
"""Creates new log object.
:param str application_type: origin of the log
:param dict dimensions: dictionary of dimensions (any data sent to api)
:param stream payload: stream to read log entry from
:param str content_type: actual content type used to send data to
server
:param bool validate: by default True, marks if log should be validated
:return: log object
:rtype: dict
:keyword: log_object
"""
payload = rest_utils.read_body(payload, content_type)
if not payload:
@ -182,6 +245,43 @@ class LogCreator(object):
return log_object
def new_log_envelope(self, log_object, tenant_id):
"""Creates new log envelope.
Log envelope is combined ouf of following properties
* log - dict
* creation_time - timestamp
* meta - meta block
Example output json would like this:
.. code-block:: json
{
"log": {
"message": "Some message",
"application_type": "monasca-log-api",
"dimension": {
"hostname": "devstack"
}
},
"creation_time": 1447834886,
"meta": {
"tenantId": "e4bd29509eda473092d32aadfee3e7b1",
"region": "pl"
}
}
:param dict log_object: log object created with
:py:meth:`LogCreator.new_log`
:param str tenant_id: ID of the tenant
:return: log envelope object
:rtype: dict
:keyword: log_envelope
"""
if not log_object:
raise LogEnvelopeException('Envelope cannot be '
'created without log')

View File

@ -40,6 +40,7 @@ VERSIONS = {
class Versions(versions_api.VersionsAPI):
"""Versions Api V2."""
def __init__(self):
super(Versions, self).__init__()