x/stackalytics
Code Issues Proposed changes

Split config definitions to make CLI help cleaner

Browse Source 
Change-Id: I2903a3e61597913c6ace84605879b06526ad07f7
This commit is contained in:
Ilya Shakhat 2015-06-22 15:54:18 +03:00
parent 13dc944373
commit 47a809bc11
8 changed files with 67 additions and 160 deletions

76
doc/source/tools/stackalytics-dump.txt

@ -1,40 +1,17 @@
usage: stackalytics-dump [-h] [--age-warn AGE_WARN]
[--collect-profiler-stats COLLECT_PROFILER_STATS]
[--config-dir DIR] [--config-file PATH]
[--corrections-uri CORRECTIONS_URI]
[--dashboard-update-interval DASHBOARD_UPDATE_INTERVAL]
[--days_to_update_members DAYS_TO_UPDATE_MEMBERS]
[--debug] [--default-metric DEFAULT_METRIC]
[--default-project-type DEFAULT_PROJECT_TYPE]
[--default-release DEFAULT_RELEASE]
[--driverlog-data-uri DRIVERLOG_DATA_URI]
[--file FILE] [--force-update]
[--git-base-uri GIT_BASE_URI]
[--listen-host LISTEN_HOST]
[--listen-port LISTEN_PORT]
[--log-config-append PATH]
usage: stackalytics-dump [-h] [--config-dir DIR] [--config-file PATH]
[--debug] [--file FILE] [--log-config-append PATH]
[--log-date-format DATE_FORMAT] [--log-dir LOG_DIR]
[--log-file PATH] [--log-format FORMAT]
[--members-look-ahead MEMBERS_LOOK_AHEAD]
[--min-compress-len MIN_COMPRESS_LEN] [--nodebug]
[--noforce-update] [--norestore] [--nouse-syslog]
[--nouse-syslog-rfc-format] [--noverbose]
[--project-list-uri PROJECT_LIST_URI] [--restore]
[--review-uri REVIEW_URI]
[--runtime-storage-uri RUNTIME_STORAGE_URI]
[--sources-root SOURCES_ROOT]
[--ssh-key-filename SSH_KEY_FILENAME]
[--ssh-username SSH_USERNAME]
[--norestore] [--nouse-syslog]
[--nouse-syslog-rfc-format] [--noverbose] [--restore]
[--syslog-log-facility SYSLOG_LOG_FACILITY]
[--use-syslog] [--use-syslog-rfc-format] [--verbose]
[--version] [--default-data-uri DEFAULT_DATA_URI]
[--version]
[--runtime-storage-uri RUNTIME_STORAGE_URI]
optional arguments:
-h, --help show this help message and exit
--age-warn AGE_WARN Warn if the age of data is more than this value, sec
--collect-profiler-stats COLLECT_PROFILER_STATS
Name of file to store python profiler data. This
option works for dashboard only
--config-dir DIR Path to a config directory to pull *.conf files from.
This file set is sorted, so as to provide a
predictable parse order if individual options are
@ -44,31 +21,9 @@ optional arguments:
--config-file PATH Path to a config file to use. Multiple config files
can be specified, with values in later files taking
precedence. The default files used are: None.
--corrections-uri CORRECTIONS_URI
The address of file with corrections data
--dashboard-update-interval DASHBOARD_UPDATE_INTERVAL
The interval specifies how frequently dashboard should
check for updates in seconds
--days_to_update_members DAYS_TO_UPDATE_MEMBERS
Number of days to update members
--debug, -d Print debugging output (set logging level to DEBUG
instead of default WARNING level).
--default-metric DEFAULT_METRIC
Default metric
--default-project-type DEFAULT_PROJECT_TYPE
Default project type
--default-release DEFAULT_RELEASE
Default release, the most recent if not set
--driverlog-data-uri DRIVERLOG_DATA_URI
URI for default data
--file FILE, -f FILE The name of file to store data
--force-update Forcibly read default data and update records
--git-base-uri GIT_BASE_URI
git base location
--listen-host LISTEN_HOST
The address dashboard listens on
--listen-port LISTEN_PORT
The port dashboard listens on
--log-config-append PATH, --log_config PATH
The name of a logging configuration file. This file is
appended to any existing logging configuration files.
@ -88,30 +43,15 @@ optional arguments:
logging.LogRecord attributes. This option is
deprecated. Please use logging_context_format_string
and logging_default_format_string instead.
--members-look-ahead MEMBERS_LOOK_AHEAD
How many member profiles to look ahead after the last
--min-compress-len MIN_COMPRESS_LEN, -m MIN_COMPRESS_LEN
The threshold length to kick in auto-compression
--nodebug The inverse of --debug
--noforce-update The inverse of --force-update
--norestore The inverse of --restore
--nouse-syslog The inverse of --use-syslog
--nouse-syslog-rfc-format
The inverse of --use-syslog-rfc-format
--noverbose The inverse of --verbose
--project-list-uri PROJECT_LIST_URI
The address of file with the official projects list
--restore, -r Restore data into memcached
--review-uri REVIEW_URI
URI of review system
--runtime-storage-uri RUNTIME_STORAGE_URI
Storage URI
--sources-root SOURCES_ROOT
The folder that holds all project sources to analyze
--ssh-key-filename SSH_KEY_FILENAME
SSH key for gerrit review system access
--ssh-username SSH_USERNAME
SSH username for gerrit review system access
--syslog-log-facility SYSLOG_LOG_FACILITY
Syslog facility to receive log lines.
--use-syslog Use syslog for logging. Existing syslog format is
@ -126,5 +66,5 @@ optional arguments:
--verbose, -v Print more verbose output (set logging level to INFO
instead of default WARNING level).
--version show program's version number and exit
--default-data-uri DEFAULT_DATA_URI
URI for default data
--runtime-storage-uri RUNTIME_STORAGE_URI
Storage URI

45
doc/source/tools/stackalytics-processor.txt

@ -1,40 +1,28 @@
usage: stackalytics-processor [-h] [--age-warn AGE_WARN]
[--collect-profiler-stats COLLECT_PROFILER_STATS]
[--config-dir DIR] [--config-file PATH]
usage: stackalytics-processor [-h] [--config-dir DIR] [--config-file PATH]
[--corrections-uri CORRECTIONS_URI]
[--dashboard-update-interval DASHBOARD_UPDATE_INTERVAL]
[--days_to_update_members DAYS_TO_UPDATE_MEMBERS]
[--debug] [--default-metric DEFAULT_METRIC]
[--default-project-type DEFAULT_PROJECT_TYPE]
[--default-release DEFAULT_RELEASE]
[--debug] [--default-data-uri DEFAULT_DATA_URI]
[--driverlog-data-uri DRIVERLOG_DATA_URI]
[--force-update] [--git-base-uri GIT_BASE_URI]
[--listen-host LISTEN_HOST]
[--listen-port LISTEN_PORT]
[--git-base-uri GIT_BASE_URI]
[--log-config-append PATH]
[--log-date-format DATE_FORMAT]
[--log-dir LOG_DIR] [--log-file PATH]
[--log-format FORMAT]
[--members-look-ahead MEMBERS_LOOK_AHEAD]
[--nodebug] [--noforce-update] [--nouse-syslog]
[--nodebug] [--nouse-syslog]
[--nouse-syslog-rfc-format] [--noverbose]
[--project-list-uri PROJECT_LIST_URI]
[--review-uri REVIEW_URI]
[--runtime-storage-uri RUNTIME_STORAGE_URI]
[--sources-root SOURCES_ROOT]
[--ssh-key-filename SSH_KEY_FILENAME]
[--ssh-username SSH_USERNAME]
[--syslog-log-facility SYSLOG_LOG_FACILITY]
[--use-syslog] [--use-syslog-rfc-format]
[--verbose] [--version]
[--default-data-uri DEFAULT_DATA_URI]
[--runtime-storage-uri RUNTIME_STORAGE_URI]
optional arguments:
-h, --help show this help message and exit
--age-warn AGE_WARN Warn if the age of data is more than this value, sec
--collect-profiler-stats COLLECT_PROFILER_STATS
Name of file to store python profiler data. This
option works for dashboard only
--config-dir DIR Path to a config directory to pull *.conf files from.
This file set is sorted, so as to provide a
predictable parse order if individual options are
@ -46,28 +34,16 @@ optional arguments:
precedence. The default files used are: None.
--corrections-uri CORRECTIONS_URI
The address of file with corrections data
--dashboard-update-interval DASHBOARD_UPDATE_INTERVAL
The interval specifies how frequently dashboard should
check for updates in seconds
--days_to_update_members DAYS_TO_UPDATE_MEMBERS
Number of days to update members
--debug, -d Print debugging output (set logging level to DEBUG
instead of default WARNING level).
--default-metric DEFAULT_METRIC
Default metric
--default-project-type DEFAULT_PROJECT_TYPE
Default project type
--default-release DEFAULT_RELEASE
Default release, the most recent if not set
--default-data-uri DEFAULT_DATA_URI
URI for default data
--driverlog-data-uri DRIVERLOG_DATA_URI
URI for default data
--force-update Forcibly read default data and update records
--git-base-uri GIT_BASE_URI
git base location
--listen-host LISTEN_HOST
The address dashboard listens on
--listen-port LISTEN_PORT
The port dashboard listens on
--log-config-append PATH, --log_config PATH
The name of a logging configuration file. This file is
appended to any existing logging configuration files.
@ -90,7 +66,6 @@ optional arguments:
--members-look-ahead MEMBERS_LOOK_AHEAD
How many member profiles to look ahead after the last
--nodebug The inverse of --debug
--noforce-update The inverse of --force-update
--nouse-syslog The inverse of --use-syslog
--nouse-syslog-rfc-format
The inverse of --use-syslog-rfc-format
@ -99,8 +74,6 @@ optional arguments:
The address of file with the official projects list
--review-uri REVIEW_URI
URI of review system
--runtime-storage-uri RUNTIME_STORAGE_URI
Storage URI
--sources-root SOURCES_ROOT
The folder that holds all project sources to analyze
--ssh-key-filename SSH_KEY_FILENAME
@ -121,5 +94,5 @@ optional arguments:
--verbose, -v Print more verbose output (set logging level to INFO
instead of default WARNING level).
--version show program's version number and exit
--default-data-uri DEFAULT_DATA_URI
URI for default data
--runtime-storage-uri RUNTIME_STORAGE_URI
Storage URI

27
etc/stackalytics.conf

@ -92,21 +92,15 @@
# From stackalytics.processor.config
#
# Storage URI (string value)
#runtime_storage_uri = memcached://127.0.0.1:11211
# URI for default data (string value)
#default_data_uri = https://git.openstack.org/cgit/stackforge/stackalytics/plain/etc/default_data.json
# The folder that holds all project sources to analyze (string value)
#sources_root = /var/local/stackalytics
# Storage URI (string value)
#runtime_storage_uri = memcached://127.0.0.1:11211
# The address dashboard listens on (string value)
#listen_host = 127.0.0.1
# The port dashboard listens on (integer value)
#listen_port = 8080
# Number of days to update members (integer value)
#days_to_update_members = 30
@ -125,15 +119,21 @@
# SSH username for gerrit review system access (string value)
#ssh_username = user
# Forcibly read default data and update records (boolean value)
#force_update = false
# The address of file with the official projects list (string value)
#project_list_uri = https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml
# URI for default data (string value)
#driverlog_data_uri = https://git.openstack.org/cgit/stackforge/driverlog/plain/etc/default_data.json
# How many member profiles to look ahead after the last (integer value)
#members_look_ahead = 250
# The address dashboard listens on (string value)
#listen_host = 127.0.0.1
# The port dashboard listens on (integer value)
#listen_port = 8080
# Default metric (string value)
#default_metric = marks
@ -151,8 +151,5 @@
# only (string value)
#collect_profiler_stats = <None>
# How many member profiles to look ahead after the last (integer value)
#members_look_ahead = 250
# Warn if the age of data is more than this value, sec (integer value)
#age_warn = 172800

2
stackalytics/dashboard/web.py

@ -43,7 +43,7 @@ app.register_blueprint(kpi.blueprint)
LOG = logging.getLogger(__name__)
conf = cfg.CONF
conf.register_opts(config.OPTS)
conf.register_opts(config.CONNECTION_OPTS + config.DASHBOARD_OPTS)
# Handlers ---------

32
stackalytics/processor/config.py

@ -18,19 +18,18 @@ import copy
from oslo_config import cfg
OPTS = [
CONNECTION_OPTS = [
cfg.StrOpt('runtime-storage-uri', default='memcached://127.0.0.1:11211',
help='Storage URI'),
]
PROCESSOR_OPTS = [
cfg.StrOpt('default-data-uri',
default='https://git.openstack.org/cgit/'
'stackforge/stackalytics/plain/etc/default_data.json',
help='URI for default data'),
cfg.StrOpt('sources-root', default='/var/local/stackalytics',
help='The folder that holds all project sources to analyze'),
cfg.StrOpt('runtime-storage-uri', default='memcached://127.0.0.1:11211',
help='Storage URI'),
cfg.StrOpt('listen-host', default='127.0.0.1',
help='The address dashboard listens on'),
cfg.IntOpt('listen-port', default=8080,
help='The port dashboard listens on'),
cfg.IntOpt('days_to_update_members', default=30,
help='Number of days to update members'),
cfg.StrOpt('corrections-uri',
@ -45,8 +44,6 @@ OPTS = [
help='SSH key for gerrit review system access'),
cfg.StrOpt('ssh-username', default='user',
help='SSH username for gerrit review system access'),
cfg.BoolOpt('force-update', default=False,
help='Forcibly read default data and update records'),
cfg.StrOpt('project-list-uri',
default=('https://git.openstack.org/cgit/'
'openstack/governance/plain/reference/projects.yaml'),
@ -55,6 +52,15 @@ OPTS = [
default='https://git.openstack.org/cgit/'
'stackforge/driverlog/plain/etc/default_data.json',
help='URI for default data'),
cfg.IntOpt('members-look-ahead', default=250,
help='How many member profiles to look ahead after the last'),
]
DASHBOARD_OPTS = [
cfg.StrOpt('listen-host', default='127.0.0.1',
help='The address dashboard listens on'),
cfg.IntOpt('listen-port', default=8080,
help='The port dashboard listens on'),
cfg.StrOpt('default-metric', default='marks',
help='Default metric'),
cfg.StrOpt('default-release',
@ -65,14 +71,12 @@ OPTS = [
help='The interval specifies how frequently dashboard should '
'check for updates in seconds'),
cfg.StrOpt('collect-profiler-stats',
help='Name of file to store python profiler data. This option '
'works for dashboard only'),
cfg.IntOpt('members-look-ahead', default=250,
help='How many member profiles to look ahead after the last'),
help='Name of file to store python profiler data'),
cfg.IntOpt('age-warn', default=2 * 24 * 60 * 60,
help='Warn if the age of data is more than this value, sec'),
]
def list_opts():
yield (None, copy.deepcopy(OPTS))
yield (None, copy.deepcopy(CONNECTION_OPTS + PROCESSOR_OPTS +
DASHBOARD_OPTS))

15
stackalytics/processor/dump.py

@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging as std_logging
import pickle
import re
import sys
@ -153,19 +152,7 @@ def _connect_to_memcached(uri):
def main():
# init conf and logging
conf = cfg.CONF
conf.register_cli_opts(config.OPTS)
conf.register_cli_opts(OPTS)
conf.register_opts(config.OPTS)
conf.register_opts(OPTS)
logging.register_options(conf)
logging.set_defaults()
conf(project='stackalytics')
logging.setup(conf, 'stackalytics')
LOG.info('Logging enabled')
conf.log_opt_values(LOG, std_logging.DEBUG)
utils.init_config_and_logging(config.CONNECTION_OPTS + OPTS)
memcached_inst = _connect_to_memcached(cfg.CONF.runtime_storage_uri)

14
stackalytics/processor/main.py

@ -14,7 +14,6 @@
# limitations under the License.
import collections
import logging as std_logging
from oslo_config import cfg
from oslo_log import log as logging
@ -319,17 +318,8 @@ def process_project_list(runtime_storage_inst, project_list_uri):
def main():
# init conf and logging
conf = cfg.CONF
conf.register_cli_opts(config.OPTS)
conf.register_opts(config.OPTS)
logging.register_options(conf)
logging.set_defaults()
conf(project='stackalytics')
logging.setup(conf, 'stackalytics')
LOG.info('Logging enabled')
conf.log_opt_values(LOG, std_logging.DEBUG)
utils.init_config_and_logging(config.CONNECTION_OPTS +
config.PROCESSOR_OPTS)
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)

16
stackalytics/processor/utils.py

@ -17,11 +17,13 @@ import cgi
import datetime
import gzip
import json
import logging as std_logging
import random
import re
import time
import iso8601
from oslo_config import cfg
from oslo_log import log as logging
import six
@ -29,6 +31,20 @@ import six
LOG = logging.getLogger(__name__)
def init_config_and_logging(opts):
conf = cfg.CONF
conf.register_cli_opts(opts)
conf.register_opts(opts)
logging.register_options(conf)
logging.set_defaults()
conf(project='stackalytics')
logging.setup(conf, 'stackalytics')
LOG.info('Logging enabled')
conf.log_opt_values(LOG, std_logging.DEBUG)
def date_to_timestamp(d):
if not d:
return 0