diff --git a/doc/source/tools/stackalytics-dump.txt b/doc/source/tools/stackalytics-dump.txt
index 6152e5408..6cc082ab2 100644
--- a/doc/source/tools/stackalytics-dump.txt
+++ b/doc/source/tools/stackalytics-dump.txt
@@ -1,40 +1,17 @@
-usage: stackalytics-dump [-h] [--age-warn AGE_WARN]
- [--collect-profiler-stats COLLECT_PROFILER_STATS]
- [--config-dir DIR] [--config-file PATH]
- [--corrections-uri CORRECTIONS_URI]
- [--dashboard-update-interval DASHBOARD_UPDATE_INTERVAL]
- [--days_to_update_members DAYS_TO_UPDATE_MEMBERS]
- [--debug] [--default-metric DEFAULT_METRIC]
- [--default-project-type DEFAULT_PROJECT_TYPE]
- [--default-release DEFAULT_RELEASE]
- [--driverlog-data-uri DRIVERLOG_DATA_URI]
- [--file FILE] [--force-update]
- [--git-base-uri GIT_BASE_URI]
- [--listen-host LISTEN_HOST]
- [--listen-port LISTEN_PORT]
- [--log-config-append PATH]
+usage: stackalytics-dump [-h] [--config-dir DIR] [--config-file PATH]
+ [--debug] [--file FILE] [--log-config-append PATH]
[--log-date-format DATE_FORMAT] [--log-dir LOG_DIR]
[--log-file PATH] [--log-format FORMAT]
- [--members-look-ahead MEMBERS_LOOK_AHEAD]
[--min-compress-len MIN_COMPRESS_LEN] [--nodebug]
- [--noforce-update] [--norestore] [--nouse-syslog]
- [--nouse-syslog-rfc-format] [--noverbose]
- [--project-list-uri PROJECT_LIST_URI] [--restore]
- [--review-uri REVIEW_URI]
- [--runtime-storage-uri RUNTIME_STORAGE_URI]
- [--sources-root SOURCES_ROOT]
- [--ssh-key-filename SSH_KEY_FILENAME]
- [--ssh-username SSH_USERNAME]
+ [--norestore] [--nouse-syslog]
+ [--nouse-syslog-rfc-format] [--noverbose] [--restore]
[--syslog-log-facility SYSLOG_LOG_FACILITY]
[--use-syslog] [--use-syslog-rfc-format] [--verbose]
- [--version] [--default-data-uri DEFAULT_DATA_URI]
+ [--version]
+ [--runtime-storage-uri RUNTIME_STORAGE_URI]
optional arguments:
-h, --help show this help message and exit
- --age-warn AGE_WARN Warn if the age of data is more than this value, sec
- --collect-profiler-stats COLLECT_PROFILER_STATS
- Name of file to store python profiler data. This
- option works for dashboard only
--config-dir DIR Path to a config directory to pull *.conf files from.
This file set is sorted, so as to provide a
predictable parse order if individual options are
@@ -44,31 +21,9 @@ optional arguments:
--config-file PATH Path to a config file to use. Multiple config files
can be specified, with values in later files taking
precedence. The default files used are: None.
- --corrections-uri CORRECTIONS_URI
- The address of file with corrections data
- --dashboard-update-interval DASHBOARD_UPDATE_INTERVAL
- The interval specifies how frequently dashboard should
- check for updates in seconds
- --days_to_update_members DAYS_TO_UPDATE_MEMBERS
- Number of days to update members
--debug, -d Print debugging output (set logging level to DEBUG
instead of default WARNING level).
- --default-metric DEFAULT_METRIC
- Default metric
- --default-project-type DEFAULT_PROJECT_TYPE
- Default project type
- --default-release DEFAULT_RELEASE
- Default release, the most recent if not set
- --driverlog-data-uri DRIVERLOG_DATA_URI
- URI for default data
--file FILE, -f FILE The name of file to store data
- --force-update Forcibly read default data and update records
- --git-base-uri GIT_BASE_URI
- git base location
- --listen-host LISTEN_HOST
- The address dashboard listens on
- --listen-port LISTEN_PORT
- The port dashboard listens on
--log-config-append PATH, --log_config PATH
The name of a logging configuration file. This file is
appended to any existing logging configuration files.
@@ -88,30 +43,15 @@ optional arguments:
logging.LogRecord attributes. This option is
deprecated. Please use logging_context_format_string
and logging_default_format_string instead.
- --members-look-ahead MEMBERS_LOOK_AHEAD
- How many member profiles to look ahead after the last
--min-compress-len MIN_COMPRESS_LEN, -m MIN_COMPRESS_LEN
The threshold length to kick in auto-compression
--nodebug The inverse of --debug
- --noforce-update The inverse of --force-update
--norestore The inverse of --restore
--nouse-syslog The inverse of --use-syslog
--nouse-syslog-rfc-format
The inverse of --use-syslog-rfc-format
--noverbose The inverse of --verbose
- --project-list-uri PROJECT_LIST_URI
- The address of file with the official projects list
--restore, -r Restore data into memcached
- --review-uri REVIEW_URI
- URI of review system
- --runtime-storage-uri RUNTIME_STORAGE_URI
- Storage URI
- --sources-root SOURCES_ROOT
- The folder that holds all project sources to analyze
- --ssh-key-filename SSH_KEY_FILENAME
- SSH key for gerrit review system access
- --ssh-username SSH_USERNAME
- SSH username for gerrit review system access
--syslog-log-facility SYSLOG_LOG_FACILITY
Syslog facility to receive log lines.
--use-syslog Use syslog for logging. Existing syslog format is
@@ -126,5 +66,5 @@ optional arguments:
--verbose, -v Print more verbose output (set logging level to INFO
instead of default WARNING level).
--version show program's version number and exit
- --default-data-uri DEFAULT_DATA_URI
- URI for default data
+ --runtime-storage-uri RUNTIME_STORAGE_URI
+ Storage URI
diff --git a/doc/source/tools/stackalytics-processor.txt b/doc/source/tools/stackalytics-processor.txt
index b961f15f1..e589bee88 100644
--- a/doc/source/tools/stackalytics-processor.txt
+++ b/doc/source/tools/stackalytics-processor.txt
@@ -1,40 +1,28 @@
-usage: stackalytics-processor [-h] [--age-warn AGE_WARN]
- [--collect-profiler-stats COLLECT_PROFILER_STATS]
- [--config-dir DIR] [--config-file PATH]
+usage: stackalytics-processor [-h] [--config-dir DIR] [--config-file PATH]
[--corrections-uri CORRECTIONS_URI]
- [--dashboard-update-interval DASHBOARD_UPDATE_INTERVAL]
[--days_to_update_members DAYS_TO_UPDATE_MEMBERS]
- [--debug] [--default-metric DEFAULT_METRIC]
- [--default-project-type DEFAULT_PROJECT_TYPE]
- [--default-release DEFAULT_RELEASE]
+ [--debug] [--default-data-uri DEFAULT_DATA_URI]
[--driverlog-data-uri DRIVERLOG_DATA_URI]
- [--force-update] [--git-base-uri GIT_BASE_URI]
- [--listen-host LISTEN_HOST]
- [--listen-port LISTEN_PORT]
+ [--git-base-uri GIT_BASE_URI]
[--log-config-append PATH]
[--log-date-format DATE_FORMAT]
[--log-dir LOG_DIR] [--log-file PATH]
[--log-format FORMAT]
[--members-look-ahead MEMBERS_LOOK_AHEAD]
- [--nodebug] [--noforce-update] [--nouse-syslog]
+ [--nodebug] [--nouse-syslog]
[--nouse-syslog-rfc-format] [--noverbose]
[--project-list-uri PROJECT_LIST_URI]
[--review-uri REVIEW_URI]
- [--runtime-storage-uri RUNTIME_STORAGE_URI]
[--sources-root SOURCES_ROOT]
[--ssh-key-filename SSH_KEY_FILENAME]
[--ssh-username SSH_USERNAME]
[--syslog-log-facility SYSLOG_LOG_FACILITY]
[--use-syslog] [--use-syslog-rfc-format]
[--verbose] [--version]
- [--default-data-uri DEFAULT_DATA_URI]
+ [--runtime-storage-uri RUNTIME_STORAGE_URI]
optional arguments:
-h, --help show this help message and exit
- --age-warn AGE_WARN Warn if the age of data is more than this value, sec
- --collect-profiler-stats COLLECT_PROFILER_STATS
- Name of file to store python profiler data. This
- option works for dashboard only
--config-dir DIR Path to a config directory to pull *.conf files from.
This file set is sorted, so as to provide a
predictable parse order if individual options are
@@ -46,28 +34,16 @@ optional arguments:
precedence. The default files used are: None.
--corrections-uri CORRECTIONS_URI
The address of file with corrections data
- --dashboard-update-interval DASHBOARD_UPDATE_INTERVAL
- The interval specifies how frequently dashboard should
- check for updates in seconds
--days_to_update_members DAYS_TO_UPDATE_MEMBERS
Number of days to update members
--debug, -d Print debugging output (set logging level to DEBUG
instead of default WARNING level).
- --default-metric DEFAULT_METRIC
- Default metric
- --default-project-type DEFAULT_PROJECT_TYPE
- Default project type
- --default-release DEFAULT_RELEASE
- Default release, the most recent if not set
+ --default-data-uri DEFAULT_DATA_URI
+ URI for default data
--driverlog-data-uri DRIVERLOG_DATA_URI
URI for default data
- --force-update Forcibly read default data and update records
--git-base-uri GIT_BASE_URI
git base location
- --listen-host LISTEN_HOST
- The address dashboard listens on
- --listen-port LISTEN_PORT
- The port dashboard listens on
--log-config-append PATH, --log_config PATH
The name of a logging configuration file. This file is
appended to any existing logging configuration files.
@@ -90,7 +66,6 @@ optional arguments:
--members-look-ahead MEMBERS_LOOK_AHEAD
How many member profiles to look ahead after the last
--nodebug The inverse of --debug
- --noforce-update The inverse of --force-update
--nouse-syslog The inverse of --use-syslog
--nouse-syslog-rfc-format
The inverse of --use-syslog-rfc-format
@@ -99,8 +74,6 @@ optional arguments:
The address of file with the official projects list
--review-uri REVIEW_URI
URI of review system
- --runtime-storage-uri RUNTIME_STORAGE_URI
- Storage URI
--sources-root SOURCES_ROOT
The folder that holds all project sources to analyze
--ssh-key-filename SSH_KEY_FILENAME
@@ -121,5 +94,5 @@ optional arguments:
--verbose, -v Print more verbose output (set logging level to INFO
instead of default WARNING level).
--version show program's version number and exit
- --default-data-uri DEFAULT_DATA_URI
- URI for default data
+ --runtime-storage-uri RUNTIME_STORAGE_URI
+ Storage URI
diff --git a/etc/stackalytics.conf b/etc/stackalytics.conf
index 964bf9f16..e74be5197 100644
--- a/etc/stackalytics.conf
+++ b/etc/stackalytics.conf
@@ -92,21 +92,15 @@
# From stackalytics.processor.config
#
+# Storage URI (string value)
+#runtime_storage_uri = memcached://127.0.0.1:11211
+
# URI for default data (string value)
#default_data_uri = https://git.openstack.org/cgit/stackforge/stackalytics/plain/etc/default_data.json
# The folder that holds all project sources to analyze (string value)
#sources_root = /var/local/stackalytics
-# Storage URI (string value)
-#runtime_storage_uri = memcached://127.0.0.1:11211
-
-# The address dashboard listens on (string value)
-#listen_host = 127.0.0.1
-
-# The port dashboard listens on (integer value)
-#listen_port = 8080
-
# Number of days to update members (integer value)
#days_to_update_members = 30
@@ -125,15 +119,21 @@
# SSH username for gerrit review system access (string value)
#ssh_username = user
-# Forcibly read default data and update records (boolean value)
-#force_update = false
-
# The address of file with the official projects list (string value)
#project_list_uri = https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml
# URI for default data (string value)
#driverlog_data_uri = https://git.openstack.org/cgit/stackforge/driverlog/plain/etc/default_data.json
+# How many member profiles to look ahead after the last (integer value)
+#members_look_ahead = 250
+
+# The address dashboard listens on (string value)
+#listen_host = 127.0.0.1
+
+# The port dashboard listens on (integer value)
+#listen_port = 8080
+
# Default metric (string value)
#default_metric = marks
@@ -151,8 +151,5 @@
# only (string value)
#collect_profiler_stats = <None>
-# How many member profiles to look ahead after the last (integer value)
-#members_look_ahead = 250
-
# Warn if the age of data is more than this value, sec (integer value)
#age_warn = 172800
diff --git a/stackalytics/dashboard/web.py b/stackalytics/dashboard/web.py
index 43e426868..4e6a3a719 100644
--- a/stackalytics/dashboard/web.py
+++ b/stackalytics/dashboard/web.py
@@ -43,7 +43,7 @@ app.register_blueprint(kpi.blueprint)
LOG = logging.getLogger(__name__)
conf = cfg.CONF
-conf.register_opts(config.OPTS)
+conf.register_opts(config.CONNECTION_OPTS + config.DASHBOARD_OPTS)
# Handlers ---------
diff --git a/stackalytics/processor/config.py b/stackalytics/processor/config.py
index 4daec5e14..59cf13e34 100644
--- a/stackalytics/processor/config.py
+++ b/stackalytics/processor/config.py
@@ -18,19 +18,18 @@ import copy
from oslo_config import cfg
-OPTS = [
+CONNECTION_OPTS = [
+ cfg.StrOpt('runtime-storage-uri', default='memcached://127.0.0.1:11211',
+ help='Storage URI'),
+]
+
+PROCESSOR_OPTS = [
cfg.StrOpt('default-data-uri',
default='https://git.openstack.org/cgit/'
'stackforge/stackalytics/plain/etc/default_data.json',
help='URI for default data'),
cfg.StrOpt('sources-root', default='/var/local/stackalytics',
help='The folder that holds all project sources to analyze'),
- cfg.StrOpt('runtime-storage-uri', default='memcached://127.0.0.1:11211',
- help='Storage URI'),
- cfg.StrOpt('listen-host', default='127.0.0.1',
- help='The address dashboard listens on'),
- cfg.IntOpt('listen-port', default=8080,
- help='The port dashboard listens on'),
cfg.IntOpt('days_to_update_members', default=30,
help='Number of days to update members'),
cfg.StrOpt('corrections-uri',
@@ -45,8 +44,6 @@ OPTS = [
help='SSH key for gerrit review system access'),
cfg.StrOpt('ssh-username', default='user',
help='SSH username for gerrit review system access'),
- cfg.BoolOpt('force-update', default=False,
- help='Forcibly read default data and update records'),
cfg.StrOpt('project-list-uri',
default=('https://git.openstack.org/cgit/'
'openstack/governance/plain/reference/projects.yaml'),
@@ -55,6 +52,15 @@ OPTS = [
default='https://git.openstack.org/cgit/'
'stackforge/driverlog/plain/etc/default_data.json',
help='URI for default data'),
+ cfg.IntOpt('members-look-ahead', default=250,
+ help='How many member profiles to look ahead after the last'),
+]
+
+DASHBOARD_OPTS = [
+ cfg.StrOpt('listen-host', default='127.0.0.1',
+ help='The address dashboard listens on'),
+ cfg.IntOpt('listen-port', default=8080,
+ help='The port dashboard listens on'),
cfg.StrOpt('default-metric', default='marks',
help='Default metric'),
cfg.StrOpt('default-release',
@@ -65,14 +71,12 @@ OPTS = [
help='The interval specifies how frequently dashboard should '
'check for updates in seconds'),
cfg.StrOpt('collect-profiler-stats',
- help='Name of file to store python profiler data. This option '
- 'works for dashboard only'),
- cfg.IntOpt('members-look-ahead', default=250,
- help='How many member profiles to look ahead after the last'),
+ help='Name of file to store python profiler data'),
cfg.IntOpt('age-warn', default=2 * 24 * 60 * 60,
help='Warn if the age of data is more than this value, sec'),
]
def list_opts():
- yield (None, copy.deepcopy(OPTS))
+ yield (None, copy.deepcopy(CONNECTION_OPTS + PROCESSOR_OPTS +
+ DASHBOARD_OPTS))
diff --git a/stackalytics/processor/dump.py b/stackalytics/processor/dump.py
index d49358985..107765d8d 100644
--- a/stackalytics/processor/dump.py
+++ b/stackalytics/processor/dump.py
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import logging as std_logging
import pickle
import re
import sys
@@ -153,19 +152,7 @@ def _connect_to_memcached(uri):
def main():
- # init conf and logging
- conf = cfg.CONF
- conf.register_cli_opts(config.OPTS)
- conf.register_cli_opts(OPTS)
- conf.register_opts(config.OPTS)
- conf.register_opts(OPTS)
- logging.register_options(conf)
- logging.set_defaults()
- conf(project='stackalytics')
-
- logging.setup(conf, 'stackalytics')
- LOG.info('Logging enabled')
- conf.log_opt_values(LOG, std_logging.DEBUG)
+ utils.init_config_and_logging(config.CONNECTION_OPTS + OPTS)
memcached_inst = _connect_to_memcached(cfg.CONF.runtime_storage_uri)
diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py
index 51085078a..24a5b13bd 100644
--- a/stackalytics/processor/main.py
+++ b/stackalytics/processor/main.py
@@ -14,7 +14,6 @@
# limitations under the License.
import collections
-import logging as std_logging
from oslo_config import cfg
from oslo_log import log as logging
@@ -319,17 +318,8 @@ def process_project_list(runtime_storage_inst, project_list_uri):
def main():
- # init conf and logging
- conf = cfg.CONF
- conf.register_cli_opts(config.OPTS)
- conf.register_opts(config.OPTS)
- logging.register_options(conf)
- logging.set_defaults()
- conf(project='stackalytics')
-
- logging.setup(conf, 'stackalytics')
- LOG.info('Logging enabled')
- conf.log_opt_values(LOG, std_logging.DEBUG)
+ utils.init_config_and_logging(config.CONNECTION_OPTS +
+ config.PROCESSOR_OPTS)
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
diff --git a/stackalytics/processor/utils.py b/stackalytics/processor/utils.py
index c9382336e..9587bcb7c 100644
--- a/stackalytics/processor/utils.py
+++ b/stackalytics/processor/utils.py
@@ -17,11 +17,13 @@ import cgi
import datetime
import gzip
import json
+import logging as std_logging
import random
import re
import time
import iso8601
+from oslo_config import cfg
from oslo_log import log as logging
import six
@@ -29,6 +31,20 @@ import six
LOG = logging.getLogger(__name__)
+def init_config_and_logging(opts):
+ conf = cfg.CONF
+ conf.register_cli_opts(opts)
+ conf.register_opts(opts)
+ logging.register_options(conf)
+ logging.set_defaults()
+
+ conf(project='stackalytics')
+
+ logging.setup(conf, 'stackalytics')
+ LOG.info('Logging enabled')
+ conf.log_opt_values(LOG, std_logging.DEBUG)
+
+
def date_to_timestamp(d):
if not d:
return 0