x/stackalytics
Code Issues Proposed changes

Split config definitions to make CLI help cleaner

Browse Source 
Change-Id: I2903a3e61597913c6ace84605879b06526ad07f7
This commit is contained in:
Ilya Shakhat 2015-06-22 15:54:18 +03:00
parent 13dc944373
commit 47a809bc11
8 changed files with 67 additions and 160 deletions

76
doc/source/tools/stackalytics-dump.txt

@ -1,40 +1,17 @@
usage: stackalytics-dump [-h] [--age-warn AGE_WARN] usage: stackalytics-dump [-h] [--config-dir DIR] [--config-file PATH]
[--collect-profiler-stats COLLECT_PROFILER_STATS] [--debug] [--file FILE] [--log-config-append PATH]
[--config-dir DIR] [--config-file PATH]
[--corrections-uri CORRECTIONS_URI]
[--dashboard-update-interval DASHBOARD_UPDATE_INTERVAL]
[--days_to_update_members DAYS_TO_UPDATE_MEMBERS]
[--debug] [--default-metric DEFAULT_METRIC]
[--default-project-type DEFAULT_PROJECT_TYPE]
[--default-release DEFAULT_RELEASE]
[--driverlog-data-uri DRIVERLOG_DATA_URI]
[--file FILE] [--force-update]
[--git-base-uri GIT_BASE_URI]
[--listen-host LISTEN_HOST]
[--listen-port LISTEN_PORT]
[--log-config-append PATH]
[--log-date-format DATE_FORMAT] [--log-dir LOG_DIR] [--log-date-format DATE_FORMAT] [--log-dir LOG_DIR]
[--log-file PATH] [--log-format FORMAT] [--log-file PATH] [--log-format FORMAT]
[--members-look-ahead MEMBERS_LOOK_AHEAD]
[--min-compress-len MIN_COMPRESS_LEN] [--nodebug] [--min-compress-len MIN_COMPRESS_LEN] [--nodebug]
[--noforce-update] [--norestore] [--nouse-syslog] [--norestore] [--nouse-syslog]
[--nouse-syslog-rfc-format] [--noverbose] [--nouse-syslog-rfc-format] [--noverbose] [--restore]
[--project-list-uri PROJECT_LIST_URI] [--restore]
[--review-uri REVIEW_URI]
[--runtime-storage-uri RUNTIME_STORAGE_URI]
[--sources-root SOURCES_ROOT]
[--ssh-key-filename SSH_KEY_FILENAME]
[--ssh-username SSH_USERNAME]
[--syslog-log-facility SYSLOG_LOG_FACILITY] [--syslog-log-facility SYSLOG_LOG_FACILITY]
[--use-syslog] [--use-syslog-rfc-format] [--verbose] [--use-syslog] [--use-syslog-rfc-format] [--verbose]
[--version] [--default-data-uri DEFAULT_DATA_URI] [--version]
[--runtime-storage-uri RUNTIME_STORAGE_URI]
optional arguments: optional arguments:
-h, --help show this help message and exit -h, --help show this help message and exit
--age-warn AGE_WARN Warn if the age of data is more than this value, sec
--collect-profiler-stats COLLECT_PROFILER_STATS
Name of file to store python profiler data. This
option works for dashboard only
--config-dir DIR Path to a config directory to pull *.conf files from. --config-dir DIR Path to a config directory to pull *.conf files from.
This file set is sorted, so as to provide a This file set is sorted, so as to provide a
predictable parse order if individual options are predictable parse order if individual options are
@ -44,31 +21,9 @@ optional arguments:
--config-file PATH Path to a config file to use. Multiple config files --config-file PATH Path to a config file to use. Multiple config files
can be specified, with values in later files taking can be specified, with values in later files taking
precedence. The default files used are: None. precedence. The default files used are: None.
--corrections-uri CORRECTIONS_URI
The address of file with corrections data
--dashboard-update-interval DASHBOARD_UPDATE_INTERVAL
The interval specifies how frequently dashboard should
check for updates in seconds
--days_to_update_members DAYS_TO_UPDATE_MEMBERS
Number of days to update members
--debug, -d Print debugging output (set logging level to DEBUG --debug, -d Print debugging output (set logging level to DEBUG
instead of default WARNING level). instead of default WARNING level).
--default-metric DEFAULT_METRIC
Default metric
--default-project-type DEFAULT_PROJECT_TYPE
Default project type
--default-release DEFAULT_RELEASE
Default release, the most recent if not set
--driverlog-data-uri DRIVERLOG_DATA_URI
URI for default data
--file FILE, -f FILE The name of file to store data --file FILE, -f FILE The name of file to store data
--force-update Forcibly read default data and update records
--git-base-uri GIT_BASE_URI
git base location
--listen-host LISTEN_HOST
The address dashboard listens on
--listen-port LISTEN_PORT
The port dashboard listens on
--log-config-append PATH, --log_config PATH --log-config-append PATH, --log_config PATH
The name of a logging configuration file. This file is The name of a logging configuration file. This file is
appended to any existing logging configuration files. appended to any existing logging configuration files.
@ -88,30 +43,15 @@ optional arguments:
logging.LogRecord attributes. This option is logging.LogRecord attributes. This option is
deprecated. Please use logging_context_format_string deprecated. Please use logging_context_format_string
and logging_default_format_string instead. and logging_default_format_string instead.
--members-look-ahead MEMBERS_LOOK_AHEAD
How many member profiles to look ahead after the last
--min-compress-len MIN_COMPRESS_LEN, -m MIN_COMPRESS_LEN --min-compress-len MIN_COMPRESS_LEN, -m MIN_COMPRESS_LEN
The threshold length to kick in auto-compression The threshold length to kick in auto-compression
--nodebug The inverse of --debug --nodebug The inverse of --debug
--noforce-update The inverse of --force-update
--norestore The inverse of --restore --norestore The inverse of --restore
--nouse-syslog The inverse of --use-syslog --nouse-syslog The inverse of --use-syslog
--nouse-syslog-rfc-format --nouse-syslog-rfc-format
The inverse of --use-syslog-rfc-format The inverse of --use-syslog-rfc-format
--noverbose The inverse of --verbose --noverbose The inverse of --verbose
--project-list-uri PROJECT_LIST_URI
The address of file with the official projects list
--restore, -r Restore data into memcached --restore, -r Restore data into memcached
--review-uri REVIEW_URI
URI of review system
--runtime-storage-uri RUNTIME_STORAGE_URI
Storage URI
--sources-root SOURCES_ROOT
The folder that holds all project sources to analyze
--ssh-key-filename SSH_KEY_FILENAME
SSH key for gerrit review system access
--ssh-username SSH_USERNAME
SSH username for gerrit review system access
--syslog-log-facility SYSLOG_LOG_FACILITY --syslog-log-facility SYSLOG_LOG_FACILITY
Syslog facility to receive log lines. Syslog facility to receive log lines.
--use-syslog Use syslog for logging. Existing syslog format is --use-syslog Use syslog for logging. Existing syslog format is
@ -126,5 +66,5 @@ optional arguments:
--verbose, -v Print more verbose output (set logging level to INFO --verbose, -v Print more verbose output (set logging level to INFO
instead of default WARNING level). instead of default WARNING level).
--version show program's version number and exit --version show program's version number and exit
--default-data-uri DEFAULT_DATA_URI --runtime-storage-uri RUNTIME_STORAGE_URI
URI for default data Storage URI

45
doc/source/tools/stackalytics-processor.txt

@ -1,40 +1,28 @@
usage: stackalytics-processor [-h] [--age-warn AGE_WARN] usage: stackalytics-processor [-h] [--config-dir DIR] [--config-file PATH]
[--collect-profiler-stats COLLECT_PROFILER_STATS]
[--config-dir DIR] [--config-file PATH]
[--corrections-uri CORRECTIONS_URI] [--corrections-uri CORRECTIONS_URI]
[--dashboard-update-interval DASHBOARD_UPDATE_INTERVAL]
[--days_to_update_members DAYS_TO_UPDATE_MEMBERS] [--days_to_update_members DAYS_TO_UPDATE_MEMBERS]
[--debug] [--default-metric DEFAULT_METRIC] [--debug] [--default-data-uri DEFAULT_DATA_URI]
[--default-project-type DEFAULT_PROJECT_TYPE]
[--default-release DEFAULT_RELEASE]
[--driverlog-data-uri DRIVERLOG_DATA_URI] [--driverlog-data-uri DRIVERLOG_DATA_URI]
[--force-update] [--git-base-uri GIT_BASE_URI] [--git-base-uri GIT_BASE_URI]
[--listen-host LISTEN_HOST]
[--listen-port LISTEN_PORT]
[--log-config-append PATH] [--log-config-append PATH]
[--log-date-format DATE_FORMAT] [--log-date-format DATE_FORMAT]
[--log-dir LOG_DIR] [--log-file PATH] [--log-dir LOG_DIR] [--log-file PATH]
[--log-format FORMAT] [--log-format FORMAT]
[--members-look-ahead MEMBERS_LOOK_AHEAD] [--members-look-ahead MEMBERS_LOOK_AHEAD]
[--nodebug] [--noforce-update] [--nouse-syslog] [--nodebug] [--nouse-syslog]
[--nouse-syslog-rfc-format] [--noverbose] [--nouse-syslog-rfc-format] [--noverbose]
[--project-list-uri PROJECT_LIST_URI] [--project-list-uri PROJECT_LIST_URI]
[--review-uri REVIEW_URI] [--review-uri REVIEW_URI]
[--runtime-storage-uri RUNTIME_STORAGE_URI]
[--sources-root SOURCES_ROOT] [--sources-root SOURCES_ROOT]
[--ssh-key-filename SSH_KEY_FILENAME] [--ssh-key-filename SSH_KEY_FILENAME]
[--ssh-username SSH_USERNAME] [--ssh-username SSH_USERNAME]
[--syslog-log-facility SYSLOG_LOG_FACILITY] [--syslog-log-facility SYSLOG_LOG_FACILITY]
[--use-syslog] [--use-syslog-rfc-format] [--use-syslog] [--use-syslog-rfc-format]
[--verbose] [--version] [--verbose] [--version]
[--default-data-uri DEFAULT_DATA_URI] [--runtime-storage-uri RUNTIME_STORAGE_URI]
optional arguments: optional arguments:
-h, --help show this help message and exit -h, --help show this help message and exit
--age-warn AGE_WARN Warn if the age of data is more than this value, sec
--collect-profiler-stats COLLECT_PROFILER_STATS
Name of file to store python profiler data. This
option works for dashboard only
--config-dir DIR Path to a config directory to pull *.conf files from. --config-dir DIR Path to a config directory to pull *.conf files from.
This file set is sorted, so as to provide a This file set is sorted, so as to provide a
predictable parse order if individual options are predictable parse order if individual options are
@ -46,28 +34,16 @@ optional arguments:
precedence. The default files used are: None. precedence. The default files used are: None.
--corrections-uri CORRECTIONS_URI --corrections-uri CORRECTIONS_URI
The address of file with corrections data The address of file with corrections data
--dashboard-update-interval DASHBOARD_UPDATE_INTERVAL
The interval specifies how frequently dashboard should
check for updates in seconds
--days_to_update_members DAYS_TO_UPDATE_MEMBERS --days_to_update_members DAYS_TO_UPDATE_MEMBERS
Number of days to update members Number of days to update members
--debug, -d Print debugging output (set logging level to DEBUG --debug, -d Print debugging output (set logging level to DEBUG
instead of default WARNING level). instead of default WARNING level).
--default-metric DEFAULT_METRIC --default-data-uri DEFAULT_DATA_URI
Default metric URI for default data
--default-project-type DEFAULT_PROJECT_TYPE
Default project type
--default-release DEFAULT_RELEASE
Default release, the most recent if not set
--driverlog-data-uri DRIVERLOG_DATA_URI --driverlog-data-uri DRIVERLOG_DATA_URI
URI for default data URI for default data
--force-update Forcibly read default data and update records
--git-base-uri GIT_BASE_URI --git-base-uri GIT_BASE_URI
git base location git base location
--listen-host LISTEN_HOST
The address dashboard listens on
--listen-port LISTEN_PORT
The port dashboard listens on
--log-config-append PATH, --log_config PATH --log-config-append PATH, --log_config PATH
The name of a logging configuration file. This file is The name of a logging configuration file. This file is
appended to any existing logging configuration files. appended to any existing logging configuration files.
@ -90,7 +66,6 @@ optional arguments:
--members-look-ahead MEMBERS_LOOK_AHEAD --members-look-ahead MEMBERS_LOOK_AHEAD
How many member profiles to look ahead after the last How many member profiles to look ahead after the last
--nodebug The inverse of --debug --nodebug The inverse of --debug
--noforce-update The inverse of --force-update
--nouse-syslog The inverse of --use-syslog --nouse-syslog The inverse of --use-syslog
--nouse-syslog-rfc-format --nouse-syslog-rfc-format
The inverse of --use-syslog-rfc-format The inverse of --use-syslog-rfc-format
@ -99,8 +74,6 @@ optional arguments:
The address of file with the official projects list The address of file with the official projects list
--review-uri REVIEW_URI --review-uri REVIEW_URI
URI of review system URI of review system
--runtime-storage-uri RUNTIME_STORAGE_URI
Storage URI
--sources-root SOURCES_ROOT --sources-root SOURCES_ROOT
The folder that holds all project sources to analyze The folder that holds all project sources to analyze
--ssh-key-filename SSH_KEY_FILENAME --ssh-key-filename SSH_KEY_FILENAME
@ -121,5 +94,5 @@ optional arguments:
--verbose, -v Print more verbose output (set logging level to INFO --verbose, -v Print more verbose output (set logging level to INFO
instead of default WARNING level). instead of default WARNING level).
--version show program's version number and exit --version show program's version number and exit
--default-data-uri DEFAULT_DATA_URI --runtime-storage-uri RUNTIME_STORAGE_URI
URI for default data Storage URI

27
etc/stackalytics.conf

@ -92,21 +92,15 @@
# From stackalytics.processor.config # From stackalytics.processor.config
# #
# Storage URI (string value)
#runtime_storage_uri = memcached://127.0.0.1:11211
# URI for default data (string value) # URI for default data (string value)
#default_data_uri = https://git.openstack.org/cgit/stackforge/stackalytics/plain/etc/default_data.json #default_data_uri = https://git.openstack.org/cgit/stackforge/stackalytics/plain/etc/default_data.json
# The folder that holds all project sources to analyze (string value) # The folder that holds all project sources to analyze (string value)
#sources_root = /var/local/stackalytics #sources_root = /var/local/stackalytics
# Storage URI (string value)
#runtime_storage_uri = memcached://127.0.0.1:11211
# The address dashboard listens on (string value)
#listen_host = 127.0.0.1
# The port dashboard listens on (integer value)
#listen_port = 8080
# Number of days to update members (integer value) # Number of days to update members (integer value)
#days_to_update_members = 30 #days_to_update_members = 30
@ -125,15 +119,21 @@
# SSH username for gerrit review system access (string value) # SSH username for gerrit review system access (string value)
#ssh_username = user #ssh_username = user
# Forcibly read default data and update records (boolean value)
#force_update = false
# The address of file with the official projects list (string value) # The address of file with the official projects list (string value)
#project_list_uri = https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml #project_list_uri = https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml
# URI for default data (string value) # URI for default data (string value)
#driverlog_data_uri = https://git.openstack.org/cgit/stackforge/driverlog/plain/etc/default_data.json #driverlog_data_uri = https://git.openstack.org/cgit/stackforge/driverlog/plain/etc/default_data.json
# How many member profiles to look ahead after the last (integer value)
#members_look_ahead = 250
# The address dashboard listens on (string value)
#listen_host = 127.0.0.1
# The port dashboard listens on (integer value)
#listen_port = 8080
# Default metric (string value) # Default metric (string value)
#default_metric = marks #default_metric = marks
@ -151,8 +151,5 @@
# only (string value) # only (string value)
#collect_profiler_stats = <None> #collect_profiler_stats = <None>
# How many member profiles to look ahead after the last (integer value)
#members_look_ahead = 250
# Warn if the age of data is more than this value, sec (integer value) # Warn if the age of data is more than this value, sec (integer value)
#age_warn = 172800 #age_warn = 172800

2
stackalytics/dashboard/web.py

@ -43,7 +43,7 @@ app.register_blueprint(kpi.blueprint)
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
conf = cfg.CONF conf = cfg.CONF
conf.register_opts(config.OPTS) conf.register_opts(config.CONNECTION_OPTS + config.DASHBOARD_OPTS)
# Handlers --------- # Handlers ---------

32
stackalytics/processor/config.py

@ -18,19 +18,18 @@ import copy
from oslo_config import cfg from oslo_config import cfg
OPTS = [ CONNECTION_OPTS = [
cfg.StrOpt('runtime-storage-uri', default='memcached://127.0.0.1:11211',
help='Storage URI'),
]
PROCESSOR_OPTS = [
cfg.StrOpt('default-data-uri', cfg.StrOpt('default-data-uri',
default='https://git.openstack.org/cgit/' default='https://git.openstack.org/cgit/'
'stackforge/stackalytics/plain/etc/default_data.json', 'stackforge/stackalytics/plain/etc/default_data.json',
help='URI for default data'), help='URI for default data'),
cfg.StrOpt('sources-root', default='/var/local/stackalytics', cfg.StrOpt('sources-root', default='/var/local/stackalytics',
help='The folder that holds all project sources to analyze'), help='The folder that holds all project sources to analyze'),
cfg.StrOpt('runtime-storage-uri', default='memcached://127.0.0.1:11211',
help='Storage URI'),
cfg.StrOpt('listen-host', default='127.0.0.1',
help='The address dashboard listens on'),
cfg.IntOpt('listen-port', default=8080,
help='The port dashboard listens on'),
cfg.IntOpt('days_to_update_members', default=30, cfg.IntOpt('days_to_update_members', default=30,
help='Number of days to update members'), help='Number of days to update members'),
cfg.StrOpt('corrections-uri', cfg.StrOpt('corrections-uri',
@ -45,8 +44,6 @@ OPTS = [
help='SSH key for gerrit review system access'), help='SSH key for gerrit review system access'),
cfg.StrOpt('ssh-username', default='user', cfg.StrOpt('ssh-username', default='user',
help='SSH username for gerrit review system access'), help='SSH username for gerrit review system access'),
cfg.BoolOpt('force-update', default=False,
help='Forcibly read default data and update records'),
cfg.StrOpt('project-list-uri', cfg.StrOpt('project-list-uri',
default=('https://git.openstack.org/cgit/' default=('https://git.openstack.org/cgit/'
'openstack/governance/plain/reference/projects.yaml'), 'openstack/governance/plain/reference/projects.yaml'),
@ -55,6 +52,15 @@ OPTS = [
default='https://git.openstack.org/cgit/' default='https://git.openstack.org/cgit/'
'stackforge/driverlog/plain/etc/default_data.json', 'stackforge/driverlog/plain/etc/default_data.json',
help='URI for default data'), help='URI for default data'),
cfg.IntOpt('members-look-ahead', default=250,
help='How many member profiles to look ahead after the last'),
]
DASHBOARD_OPTS = [
cfg.StrOpt('listen-host', default='127.0.0.1',
help='The address dashboard listens on'),
cfg.IntOpt('listen-port', default=8080,
help='The port dashboard listens on'),
cfg.StrOpt('default-metric', default='marks', cfg.StrOpt('default-metric', default='marks',
help='Default metric'), help='Default metric'),
cfg.StrOpt('default-release', cfg.StrOpt('default-release',
@ -65,14 +71,12 @@ OPTS = [
help='The interval specifies how frequently dashboard should ' help='The interval specifies how frequently dashboard should '
'check for updates in seconds'), 'check for updates in seconds'),
cfg.StrOpt('collect-profiler-stats', cfg.StrOpt('collect-profiler-stats',
help='Name of file to store python profiler data. This option ' help='Name of file to store python profiler data'),
'works for dashboard only'),
cfg.IntOpt('members-look-ahead', default=250,
help='How many member profiles to look ahead after the last'),
cfg.IntOpt('age-warn', default=2 * 24 * 60 * 60, cfg.IntOpt('age-warn', default=2 * 24 * 60 * 60,
help='Warn if the age of data is more than this value, sec'), help='Warn if the age of data is more than this value, sec'),
] ]
def list_opts(): def list_opts():
yield (None, copy.deepcopy(OPTS)) yield (None, copy.deepcopy(CONNECTION_OPTS + PROCESSOR_OPTS +
DASHBOARD_OPTS))

15
stackalytics/processor/dump.py

@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging as std_logging
import pickle import pickle
import re import re
import sys import sys
@ -153,19 +152,7 @@ def _connect_to_memcached(uri):
def main(): def main():
# init conf and logging utils.init_config_and_logging(config.CONNECTION_OPTS + OPTS)
conf = cfg.CONF
conf.register_cli_opts(config.OPTS)
conf.register_cli_opts(OPTS)
conf.register_opts(config.OPTS)
conf.register_opts(OPTS)
logging.register_options(conf)
logging.set_defaults()
conf(project='stackalytics')
logging.setup(conf, 'stackalytics')
LOG.info('Logging enabled')
conf.log_opt_values(LOG, std_logging.DEBUG)
memcached_inst = _connect_to_memcached(cfg.CONF.runtime_storage_uri) memcached_inst = _connect_to_memcached(cfg.CONF.runtime_storage_uri)

14
stackalytics/processor/main.py

@ -14,7 +14,6 @@
# limitations under the License. # limitations under the License.
import collections import collections
import logging as std_logging
from oslo_config import cfg from oslo_config import cfg
from oslo_log import log as logging from oslo_log import log as logging
@ -319,17 +318,8 @@ def process_project_list(runtime_storage_inst, project_list_uri):
def main(): def main():
# init conf and logging utils.init_config_and_logging(config.CONNECTION_OPTS +
conf = cfg.CONF config.PROCESSOR_OPTS)
conf.register_cli_opts(config.OPTS)
conf.register_opts(config.OPTS)
logging.register_options(conf)
logging.set_defaults()
conf(project='stackalytics')
logging.setup(conf, 'stackalytics')
LOG.info('Logging enabled')
conf.log_opt_values(LOG, std_logging.DEBUG)
runtime_storage_inst = runtime_storage.get_runtime_storage( runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri) cfg.CONF.runtime_storage_uri)

16
stackalytics/processor/utils.py

@ -17,11 +17,13 @@ import cgi
import datetime import datetime
import gzip import gzip
import json import json
import logging as std_logging
import random import random
import re import re
import time import time
import iso8601 import iso8601
from oslo_config import cfg
from oslo_log import log as logging from oslo_log import log as logging
import six import six
@ -29,6 +31,20 @@ import six
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
def init_config_and_logging(opts):
conf = cfg.CONF
conf.register_cli_opts(opts)
conf.register_opts(opts)
logging.register_options(conf)
logging.set_defaults()
conf(project='stackalytics')
logging.setup(conf, 'stackalytics')
LOG.info('Logging enabled')
conf.log_opt_values(LOG, std_logging.DEBUG)
def date_to_timestamp(d): def date_to_timestamp(d):
if not d: if not d:
return 0 return 0