From 7cdf1408f07f6033a48f9271f97658ae7ea9045d Mon Sep 17 00:00:00 2001 From: Ilya Shakhat Date: Mon, 4 Jul 2016 17:16:52 +0300 Subject: [PATCH] Make Gerrit retry counter configurable A new config parameter 'gerrit-retry' is introduced with default value 10. Change-Id: I2b7fcd704873cacd147026e9f3bb38e9752cb435 --- doc/source/tools/stackalytics-processor.txt | 5 ++++- etc/stackalytics.conf | 5 ++++- stackalytics/processor/config.py | 2 ++ stackalytics/processor/main.py | 3 ++- stackalytics/processor/rcs.py | 6 ++++-- 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/doc/source/tools/stackalytics-processor.txt b/doc/source/tools/stackalytics-processor.txt index 4bb62a4bc..8ad6f6169 100644 --- a/doc/source/tools/stackalytics-processor.txt +++ b/doc/source/tools/stackalytics-processor.txt @@ -3,6 +3,7 @@ usage: stackalytics-processor [-h] [--config-dir DIR] [--config-file PATH] [--days_to_update_members DAYS_TO_UPDATE_MEMBERS] [--debug] [--default-data-uri DEFAULT_DATA_URI] [--driverlog-data-uri DRIVERLOG_DATA_URI] + [--gerrit-retry GERRIT_RETRY] [--git-base-uri GIT_BASE_URI] [--log-config-append PATH] [--log-date-format DATE_FORMAT] @@ -44,6 +45,8 @@ optional arguments: URI for default data --driverlog-data-uri DRIVERLOG_DATA_URI URI for default data + --gerrit-retry GERRIT_RETRY + How many times to retry after Gerrit errors --git-base-uri GIT_BASE_URI git base location --log-config-append PATH, --log_config PATH @@ -98,7 +101,7 @@ optional arguments: Syslog facility to receive log lines. This option is ignored if log_config_append is set. --translation-team-uri TRANSLATION_TEAM_URI - URI for translation team data + URI of translation team data --use-syslog Use syslog for logging. Existing syslog format is DEPRECATED and will be changed later to honor RFC5424. This option is ignored if log_config_append is set. diff --git a/etc/stackalytics.conf b/etc/stackalytics.conf index 7073d338a..4b026ee61 100644 --- a/etc/stackalytics.conf +++ b/etc/stackalytics.conf @@ -144,7 +144,7 @@ # URI for default data (string value) #driverlog_data_uri = https://git.openstack.org/cgit/openstack/driverlog/plain/etc/default_data.json -# URI for translation team data (string value) +# URI of translation team data (string value) #translation_team_uri = https://git.openstack.org/cgit/openstack/i18n/plain/tools/zanata/translation_team.yaml # How many member profiles to look ahead after the last (integer value) @@ -153,6 +153,9 @@ # Number of seconds to wait for remote response (integer value) #read_timeout = 120 +# How many times to retry after Gerrit errors (integer value) +#gerrit_retry = 10 + # The address dashboard listens on (string value) #listen_host = 127.0.0.1 diff --git a/stackalytics/processor/config.py b/stackalytics/processor/config.py index 358322e59..4b720e392 100644 --- a/stackalytics/processor/config.py +++ b/stackalytics/processor/config.py @@ -56,6 +56,8 @@ PROCESSOR_OPTS = [ help='How many member profiles to look ahead after the last'), cfg.IntOpt('read-timeout', default=120, help='Number of seconds to wait for remote response'), + cfg.IntOpt('gerrit-retry', default=10, + help='How many times to retry after Gerrit errors'), ] DASHBOARD_OPTS = [ diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py index 214ff684b..5ab8a258f 100644 --- a/stackalytics/processor/main.py +++ b/stackalytics/processor/main.py @@ -223,7 +223,8 @@ def process(runtime_storage_inst, record_processor_inst): rcs_inst = rcs.get_rcs(cfg.CONF.review_uri) rcs_inst.setup(key_filename=cfg.CONF.ssh_key_filename, - username=cfg.CONF.ssh_username) + username=cfg.CONF.ssh_username, + gerrit_retry=cfg.CONF.gerrit_retry) for repo in repos: _process_repo(repo, runtime_storage_inst, record_processor_inst, diff --git a/stackalytics/processor/rcs.py b/stackalytics/processor/rcs.py index 012d1302b..fa8793970 100644 --- a/stackalytics/processor/rcs.py +++ b/stackalytics/processor/rcs.py @@ -26,7 +26,7 @@ DEFAULT_PORT = 29418 GERRIT_URI_PREFIX = r'^gerrit:\/\/' PAGE_LIMIT = 100 REQUEST_COUNT_LIMIT = 20 -SSH_ERRORS_LIMIT = 5 +SSH_ERRORS_LIMIT = 10 class RcsException(Exception): @@ -65,6 +65,7 @@ class Gerrit(Rcs): self.key_filename = None self.username = None + self.ssh_errors_limit = SSH_ERRORS_LIMIT self.client = paramiko.SSHClient() self.client.load_system_host_keys() @@ -76,6 +77,7 @@ class Gerrit(Rcs): def setup(self, **kwargs): self.key_filename = kwargs.get('key_filename') self.username = kwargs.get('username') + self.ssh_errors_limit = kwargs.get('gerrit_retry') or SSH_ERRORS_LIMIT self._connect() @@ -123,7 +125,7 @@ class Gerrit(Rcs): raise RcsException(e) def _exec_command_with_retrial(self, cmd): - while self.error_count < SSH_ERRORS_LIMIT: + while self.error_count < self.ssh_errors_limit: try: return self._exec_command(cmd) except RcsException: