x
/
stackalytics
Code Issues Proposed changes

Enable config auto-generation 

Change-Id: I561f4264bfad614cffa8e768cdfd1eb87f0e21c1
This commit is contained in:
Ilya Shakhat 2015-04-01 18:54:29 +03:00
parent ce121e8067
commit 5761a9e584
11 changed files with 539 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
config-generator.conf Normal file
View File

@ -0,0 +1,5 @@
[DEFAULT]
output_file = etc/stackalytics.conf
wrap_width = 79
namespace = stackalytics.processor.config
namespace = oslo_log

9
doc/source/index.rst
View File

@ -14,13 +14,20 @@ Overview
API
----------
---
.. toctree::
:maxdepth: 1
userdoc/api_v1.0
CLI Tools
---------
.. toctree::
:maxdepth: 1
tools.rst
Developer Guide
---------------
**Other Resources**

6
doc/source/tools.rst Normal file
View File

@ -0,0 +1,6 @@
stackalytics-processor
----------------------
Run Stackalytics processor
.. literalinclude:: tools/stackalytics-processor.txt

61
doc/source/tools/stackalytics-dashboard.txt Normal file
View File

@ -0,0 +1,61 @@
usage: stackalytics-dashboard [-h] [--config-dir DIR] [--config-file PATH]
[--debug] [--log-config-append PATH]
[--log-date-format DATE_FORMAT]
[--log-dir LOG_DIR] [--log-file PATH]
[--log-format FORMAT] [--nouse-syslog]
[--nouse-syslog-rfc-format] [--noverbose]
[--syslog-log-facility SYSLOG_LOG_FACILITY]
[--use-syslog] [--use-syslog-rfc-format]
[--verbose] [--version] [--nodebug]
optional arguments:
-h, --help show this help message and exit
--config-dir DIR Path to a config directory to pull *.conf files from.
This file set is sorted, so as to provide a
predictable parse order if individual options are
over-ridden. The set is parsed after the file(s)
specified via previous --config-file, arguments hence
over-ridden options in the directory take precedence.
--config-file PATH Path to a config file to use. Multiple config files
can be specified, with values in later files taking
precedence. The default files used are: None.
--debug, -d Print debugging output (set logging level to DEBUG
instead of default WARNING level).
--log-config-append PATH, --log_config PATH
The name of a logging configuration file. This file is
appended to any existing logging configuration files.
For details about logging configuration files, see the
Python logging module documentation.
--log-date-format DATE_FORMAT
Format string for %(asctime)s in log records. Default:
None .
--log-dir LOG_DIR, --logdir LOG_DIR
(Optional) The base directory used for relative --log-
file paths.
--log-file PATH, --logfile PATH
(Optional) Name of log file to output to. If no
default is set, logging will go to stdout.
--log-format FORMAT DEPRECATED. A logging.Formatter log message format
string which may use any of the available
logging.LogRecord attributes. This option is
deprecated. Please use logging_context_format_string
and logging_default_format_string instead.
--nouse-syslog The inverse of --use-syslog
--nouse-syslog-rfc-format
The inverse of --use-syslog-rfc-format
--noverbose The inverse of --verbose
--syslog-log-facility SYSLOG_LOG_FACILITY
Syslog facility to receive log lines.
--use-syslog Use syslog for logging. Existing syslog format is
DEPRECATED during I, and will change in J to honor
RFC5424.
--use-syslog-rfc-format
(Optional) Enables or disables syslog rfc5424 format
for logging. If enabled, prefixes the MSG part of the
syslog message with APP-NAME (RFC5424). The format
without the APP-NAME is deprecated in I, and will be
removed in J.
--verbose, -v Print more verbose output (set logging level to INFO
instead of default WARNING level).
--version show program's version number and exit
--nodebug The inverse of --debug

129
doc/source/tools/stackalytics-dump.txt Normal file
View File

@ -0,0 +1,129 @@
usage: stackalytics-dump [-h]
[--collect-profiler-stats COLLECT_PROFILER_STATS]
[--config-dir DIR] [--config-file PATH]
[--corrections-uri CORRECTIONS_URI]
[--dashboard-update-interval DASHBOARD_UPDATE_INTERVAL]
[--days_to_update_members DAYS_TO_UPDATE_MEMBERS]
[--debug] [--default-metric DEFAULT_METRIC]
[--default-project-type DEFAULT_PROJECT_TYPE]
[--default-release DEFAULT_RELEASE]
[--driverlog-data-uri DRIVERLOG_DATA_URI]
[--file FILE] [--force-update]
[--git-base-uri GIT_BASE_URI]
[--listen-host LISTEN_HOST]
[--listen-port LISTEN_PORT]
[--log-config-append PATH]
[--log-date-format DATE_FORMAT] [--log-dir LOG_DIR]
[--log-file PATH] [--log-format FORMAT]
[--members-look-ahead MEMBERS_LOOK_AHEAD]
[--min-compress-len MIN_COMPRESS_LEN] [--nodebug]
[--noforce-update] [--norestore] [--nouse-syslog]
[--nouse-syslog-rfc-format] [--noverbose]
[--project-list-uri PROJECT_LIST_URI] [--restore]
[--review-uri REVIEW_URI]
[--runtime-storage-uri RUNTIME_STORAGE_URI]
[--sources-root SOURCES_ROOT]
[--ssh-key-filename SSH_KEY_FILENAME]
[--ssh-username SSH_USERNAME]
[--syslog-log-facility SYSLOG_LOG_FACILITY]
[--use-syslog] [--use-syslog-rfc-format] [--verbose]
[--version] [--default-data-uri DEFAULT_DATA_URI]
optional arguments:
-h, --help show this help message and exit
--collect-profiler-stats COLLECT_PROFILER_STATS
Name of file to store python profiler data. This
option works for dashboard only
--config-dir DIR Path to a config directory to pull *.conf files from.
This file set is sorted, so as to provide a
predictable parse order if individual options are
over-ridden. The set is parsed after the file(s)
specified via previous --config-file, arguments hence
over-ridden options in the directory take precedence.
--config-file PATH Path to a config file to use. Multiple config files
can be specified, with values in later files taking
precedence. The default files used are: None.
--corrections-uri CORRECTIONS_URI
The address of file with corrections data
--dashboard-update-interval DASHBOARD_UPDATE_INTERVAL
The interval specifies how frequently dashboard should
check for updates in seconds
--days_to_update_members DAYS_TO_UPDATE_MEMBERS
Number of days to update members
--debug, -d Print debugging output (set logging level to DEBUG
instead of default WARNING level).
--default-metric DEFAULT_METRIC
Default metric
--default-project-type DEFAULT_PROJECT_TYPE
Default project type
--default-release DEFAULT_RELEASE
Default release, the most recent if not set
--driverlog-data-uri DRIVERLOG_DATA_URI
URI for default data
--file FILE, -f FILE The name of file to store data
--force-update Forcibly read default data and update records
--git-base-uri GIT_BASE_URI
git base location
--listen-host LISTEN_HOST
The address dashboard listens on
--listen-port LISTEN_PORT
The port dashboard listens on
--log-config-append PATH, --log_config PATH
The name of a logging configuration file. This file is
appended to any existing logging configuration files.
For details about logging configuration files, see the
Python logging module documentation.
--log-date-format DATE_FORMAT
Format string for %(asctime)s in log records. Default:
None .
--log-dir LOG_DIR, --logdir LOG_DIR
(Optional) The base directory used for relative --log-
file paths.
--log-file PATH, --logfile PATH
(Optional) Name of log file to output to. If no
default is set, logging will go to stdout.
--log-format FORMAT DEPRECATED. A logging.Formatter log message format
string which may use any of the available
logging.LogRecord attributes. This option is
deprecated. Please use logging_context_format_string
and logging_default_format_string instead.
--members-look-ahead MEMBERS_LOOK_AHEAD
How many member profiles to look ahead after the last
--min-compress-len MIN_COMPRESS_LEN, -m MIN_COMPRESS_LEN
The threshold length to kick in auto-compression
--nodebug The inverse of --debug
--noforce-update The inverse of --force-update
--norestore The inverse of --restore
--nouse-syslog The inverse of --use-syslog
--nouse-syslog-rfc-format
The inverse of --use-syslog-rfc-format
--noverbose The inverse of --verbose
--project-list-uri PROJECT_LIST_URI
The address of file with the official projects list
--restore, -r Restore data into memcached
--review-uri REVIEW_URI
URI of review system
--runtime-storage-uri RUNTIME_STORAGE_URI
Storage URI
--sources-root SOURCES_ROOT
The folder that holds all project sources to analyze
--ssh-key-filename SSH_KEY_FILENAME
SSH key for gerrit review system access
--ssh-username SSH_USERNAME
SSH username for gerrit review system access
--syslog-log-facility SYSLOG_LOG_FACILITY
Syslog facility to receive log lines.
--use-syslog Use syslog for logging. Existing syslog format is
DEPRECATED during I, and will change in J to honor
RFC5424.
--use-syslog-rfc-format
(Optional) Enables or disables syslog rfc5424 format
for logging. If enabled, prefixes the MSG part of the
syslog message with APP-NAME (RFC5424). The format
without the APP-NAME is deprecated in I, and will be
removed in J.
--verbose, -v Print more verbose output (set logging level to INFO
instead of default WARNING level).
--version show program's version number and exit
--default-data-uri DEFAULT_DATA_URI
URI for default data

124
doc/source/tools/stackalytics-processor.txt Normal file
View File

@ -0,0 +1,124 @@
usage: stackalytics-processor [-h]
[--collect-profiler-stats COLLECT_PROFILER_STATS]
[--config-dir DIR] [--config-file PATH]
[--corrections-uri CORRECTIONS_URI]
[--dashboard-update-interval DASHBOARD_UPDATE_INTERVAL]
[--days_to_update_members DAYS_TO_UPDATE_MEMBERS]
[--debug] [--default-metric DEFAULT_METRIC]
[--default-project-type DEFAULT_PROJECT_TYPE]
[--default-release DEFAULT_RELEASE]
[--driverlog-data-uri DRIVERLOG_DATA_URI]
[--force-update] [--git-base-uri GIT_BASE_URI]
[--listen-host LISTEN_HOST]
[--listen-port LISTEN_PORT]
[--log-config-append PATH]
[--log-date-format DATE_FORMAT]
[--log-dir LOG_DIR] [--log-file PATH]
[--log-format FORMAT]
[--members-look-ahead MEMBERS_LOOK_AHEAD]
[--nodebug] [--noforce-update] [--nouse-syslog]
[--nouse-syslog-rfc-format] [--noverbose]
[--project-list-uri PROJECT_LIST_URI]
[--review-uri REVIEW_URI]
[--runtime-storage-uri RUNTIME_STORAGE_URI]
[--sources-root SOURCES_ROOT]
[--ssh-key-filename SSH_KEY_FILENAME]
[--ssh-username SSH_USERNAME]
[--syslog-log-facility SYSLOG_LOG_FACILITY]
[--use-syslog] [--use-syslog-rfc-format]
[--verbose] [--version]
[--default-data-uri DEFAULT_DATA_URI]
optional arguments:
-h, --help show this help message and exit
--collect-profiler-stats COLLECT_PROFILER_STATS
Name of file to store python profiler data. This
option works for dashboard only
--config-dir DIR Path to a config directory to pull *.conf files from.
This file set is sorted, so as to provide a
predictable parse order if individual options are
over-ridden. The set is parsed after the file(s)
specified via previous --config-file, arguments hence
over-ridden options in the directory take precedence.
--config-file PATH Path to a config file to use. Multiple config files
can be specified, with values in later files taking
precedence. The default files used are: None.
--corrections-uri CORRECTIONS_URI
The address of file with corrections data
--dashboard-update-interval DASHBOARD_UPDATE_INTERVAL
The interval specifies how frequently dashboard should
check for updates in seconds
--days_to_update_members DAYS_TO_UPDATE_MEMBERS
Number of days to update members
--debug, -d Print debugging output (set logging level to DEBUG
instead of default WARNING level).
--default-metric DEFAULT_METRIC
Default metric
--default-project-type DEFAULT_PROJECT_TYPE
Default project type
--default-release DEFAULT_RELEASE
Default release, the most recent if not set
--driverlog-data-uri DRIVERLOG_DATA_URI
URI for default data
--force-update Forcibly read default data and update records
--git-base-uri GIT_BASE_URI
git base location
--listen-host LISTEN_HOST
The address dashboard listens on
--listen-port LISTEN_PORT
The port dashboard listens on
--log-config-append PATH, --log_config PATH
The name of a logging configuration file. This file is
appended to any existing logging configuration files.
For details about logging configuration files, see the
Python logging module documentation.
--log-date-format DATE_FORMAT
Format string for %(asctime)s in log records. Default:
None .
--log-dir LOG_DIR, --logdir LOG_DIR
(Optional) The base directory used for relative --log-
file paths.
--log-file PATH, --logfile PATH
(Optional) Name of log file to output to. If no
default is set, logging will go to stdout.
--log-format FORMAT DEPRECATED. A logging.Formatter log message format
string which may use any of the available
logging.LogRecord attributes. This option is
deprecated. Please use logging_context_format_string
and logging_default_format_string instead.
--members-look-ahead MEMBERS_LOOK_AHEAD
How many member profiles to look ahead after the last
--nodebug The inverse of --debug
--noforce-update The inverse of --force-update
--nouse-syslog The inverse of --use-syslog
--nouse-syslog-rfc-format
The inverse of --use-syslog-rfc-format
--noverbose The inverse of --verbose
--project-list-uri PROJECT_LIST_URI
The address of file with the official projects list
--review-uri REVIEW_URI
URI of review system
--runtime-storage-uri RUNTIME_STORAGE_URI
Storage URI
--sources-root SOURCES_ROOT
The folder that holds all project sources to analyze
--ssh-key-filename SSH_KEY_FILENAME
SSH key for gerrit review system access
--ssh-username SSH_USERNAME
SSH username for gerrit review system access
--syslog-log-facility SYSLOG_LOG_FACILITY
Syslog facility to receive log lines.
--use-syslog Use syslog for logging. Existing syslog format is
DEPRECATED during I, and will change in J to honor
RFC5424.
--use-syslog-rfc-format
(Optional) Enables or disables syslog rfc5424 format
for logging. If enabled, prefixes the MSG part of the
syslog message with APP-NAME (RFC5424). The format
without the APP-NAME is deprecated in I, and will be
removed in J.
--verbose, -v Print more verbose output (set logging level to INFO
instead of default WARNING level).
--version show program's version number and exit
--default-data-uri DEFAULT_DATA_URI
URI for default data

171
etc/stackalytics.conf
View File

@ -1,63 +1,150 @@
[DEFAULT]
# Run in debug mode?
# debug = False
# Default data
# default_data_uri = https://git.openstack.org/cgit/stackforge/stackalytics/plain/etc/default_data.json
#
# From oslo_log
#
# The folder that holds all project sources to analyze
# sources_root = /var/local/stackalytics
# Print debugging output (set logging level to DEBUG instead of default WARNING
# level). (boolean value)
#debug = false
# Runtime storage URI
# runtime_storage_uri = memcached://127.0.0.1:11211
# Print more verbose output (set logging level to INFO instead of default
# WARNING level). (boolean value)
#verbose = false
# Hostname where dashboard listens on
# listen_host = 127.0.0.1
# The name of a logging configuration file. This file is appended to any
# existing logging configuration files. For details about logging configuration
# files, see the Python logging module documentation. (string value)
# Deprecated group/name - [DEFAULT]/log_config
#log_config_append = <None>
# Port where dashboard listens on
# listen_port = 8080
# DEPRECATED. A logging.Formatter log message format string which may use any
# of the available logging.LogRecord attributes. This option is deprecated.
# Please use logging_context_format_string and logging_default_format_string
# instead. (string value)
#log_format = <None>
# Number of days to update members
# days_to_update_members = 30
# Format string for %%(asctime)s in log records. Default: %(default)s . (string
# value)
#log_date_format = %Y-%m-%d %H:%M:%S
# The address of file with corrections data
# corrections_uri = https://git.openstack.org/cgit/stackforge/stackalytics/plain/etc/corrections.json
# (Optional) Name of log file to output to. If no default is set, logging will
# go to stdout. (string value)
# Deprecated group/name - [DEFAULT]/logfile
#log_file = <None>
# URI of review system
# review_uri = gerrit://review.openstack.org
# (Optional) The base directory used for relative --log-file paths. (string
# value)
# Deprecated group/name - [DEFAULT]/logdir
#log_dir = <None>
# git base location
# git_base_uri = git://git.openstack.org
# Use syslog for logging. Existing syslog format is DEPRECATED during I, and
# will change in J to honor RFC5424. (boolean value)
#use_syslog = false
# SSH key for gerrit review system access
# ssh_key_filename = /home/user/.ssh/id_rsa
# (Optional) Enables or disables syslog rfc5424 format for logging. If enabled,
# prefixes the MSG part of the syslog message with APP-NAME (RFC5424). The
# format without the APP-NAME is deprecated in I, and will be removed in J.
# (boolean value)
#use_syslog_rfc_format = false
# SSH username for gerrit review system access
# ssh_username = user
# Syslog facility to receive log lines. (string value)
#syslog_log_facility = LOG_USER
# Forcibly read default data and update records
# force_update = False
# Log output to standard error. (boolean value)
#use_stderr = true
# The address of file with the official projects list
# project_list_uri = https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml
# Format string to use for log messages with context. (string value)
#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s
# The address of DriverLog data
# driverlog_data_uri = https://git.openstack.org/cgit/stackforge/driverlog/plain/etc/default_data.json
# Format string to use for log messages without context. (string value)
#logging_default_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s
# Default metric
# default_metric = marks
# Data to append to log format when level is DEBUG. (string value)
#logging_debug_format_suffix = %(funcName)s %(pathname)s:%(lineno)d
# Default release, the most recent if not set
# default_release =
# Prefix each line of exception output with this format. (string value)
#logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s
# Default project type
# default_project_type = openstack
# List of logger=LEVEL pairs. (list value)
#default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN
# The interval specifies how frequently dashboard should check for updates in seconds
# dashboard_update_interval = 3600
# Enables or disables publication of error events. (boolean value)
#publish_errors = false
# Name of file to store python profiler data. This option works for dashboard only
# collect_profiler_stats =
# Enables or disables fatal status of deprecations. (boolean value)
#fatal_deprecations = false
# How many member profiles to look ahead after the last
# members_look_ahead = 250
# The format for an instance that is passed with the log message. (string
# value)
#instance_format = "[instance: %(uuid)s] "
# The format for an instance UUID that is passed with the log message. (string
# value)
#instance_uuid_format = "[instance: %(uuid)s] "
#
# From stackalytics.processor.config
#
# URI for default data (string value)
#default_data_uri = https://git.openstack.org/cgit/stackforge/stackalytics/plain/etc/default_data.json
# The folder that holds all project sources to analyze (string value)
#sources_root = /var/local/stackalytics
# Storage URI (string value)
#runtime_storage_uri = memcached://127.0.0.1:11211
# The address dashboard listens on (string value)
#listen_host = 127.0.0.1
# The port dashboard listens on (integer value)
#listen_port = 8080
# Number of days to update members (integer value)
#days_to_update_members = 30
# The address of file with corrections data (string value)
#corrections_uri = https://git.openstack.org/cgit/stackforge/stackalytics/plain/etc/corrections.json
# URI of review system (string value)
#review_uri = gerrit://review.openstack.org
# git base location (string value)
#git_base_uri = git://git.openstack.org
# SSH key for gerrit review system access (string value)
#ssh_key_filename = /home/user/.ssh/id_rsa
# SSH username for gerrit review system access (string value)
#ssh_username = user
# Forcibly read default data and update records (boolean value)
#force_update = false
# The address of file with the official projects list (string value)
#project_list_uri = https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml
# URI for default data (string value)
#driverlog_data_uri = https://git.openstack.org/cgit/stackforge/driverlog/plain/etc/default_data.json
# Default metric (string value)
#default_metric = marks
# Default release, the most recent if not set (string value)
#default_release = <None>
# Default project type (string value)
#default_project_type = openstack
# The interval specifies how frequently dashboard should check for updates in
# seconds (integer value)
#dashboard_update_interval = 3600
# Name of file to store python profiler data. This option works for dashboard
# only (string value)
#collect_profiler_stats = <None>
# How many member profiles to look ahead after the last (integer value)
#members_look_ahead = 250

4
setup.cfg
View File

@ -32,3 +32,7 @@ console_scripts =
stackalytics-dump = stackalytics.processor.dump:main
stackalytics-dashboard = stackalytics.dashboard.web:main
stackalytics-processor = stackalytics.processor.main:main
oslo.config.opts =
oslo_log = oslo_log._options:list_opts
stackalytics.processor.config = stackalytics.processor.config:list_opts

6
stackalytics/processor/config.py
View File

@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
from oslo_config import cfg
@ -68,3 +70,7 @@ OPTS = [
cfg.IntOpt('members-look-ahead', default=250,
help='How many member profiles to look ahead after the last'),
]
def list_opts():
yield (None, copy.deepcopy(OPTS))

62
tools/cli_auto_doc.py Normal file
View File

@ -0,0 +1,62 @@
# Copyright (c) 2015 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
try:
import ConfigParser as configparser
except ImportError:
import configparser
def split_multiline(value):
value = [element for element in
(line.strip() for line in value.split('\n'))
if element]
return value
def get_entry_points(config):
if 'entry_points' not in config:
return {}
return dict((option, split_multiline(value))
for option, value in config['entry_points'].items())
def make(cfg, dest):
parser = configparser.RawConfigParser()
parser.read(cfg)
config = {}
for section in parser.sections():
config[section] = dict(parser.items(section))
entry_points = get_entry_points(config)
console_scripts = entry_points.get('console_scripts')
if console_scripts:
for item in console_scripts:
tool = item.split('=')[0].strip()
print('Running %s' % tool)
os.system('%(tool)s --help > %(dest)s/%(tool)s.txt' %
dict(tool=tool, dest=dest))
if len(sys.argv) < 2:
print('Usage: cli_auto_doc <dest folder>')
sys.exit(1)
print('Generating docs from help to console tools')
make(cfg='setup.cfg', dest=sys.argv[1])

5
tox.ini
View File

@ -29,6 +29,11 @@ commands = {posargs}
[testenv:cover]
commands = python setup.py testr --coverage --testr-args='{posargs}'
[testenv:genconfig]
commands =
oslo-config-generator --config-file=config-generator.conf
python tools/cli_auto_doc.py doc/source/tools
[tox:jenkins]
downloadcache = ~/cache/pip