From ebeee06e8eca4cbba69e255e7ef43437cf179b57 Mon Sep 17 00:00:00 2001 From: Andrii Ostapenko Date: Tue, 5 Mar 2019 14:14:44 -0600 Subject: [PATCH] Allow to query Gerrit behind corporate proxy Adds proxy_command configuration option to allow paramiko to communicate with Gerrit behind corporate proxy. Change-Id: I5418b31b15b83f55c2f78636ffa958755c55de8d --- etc/stackalytics.conf | 4 ++++ stackalytics/processor/config.py | 2 ++ stackalytics/processor/rcs.py | 14 +++++++++++- stackalytics/tests/unit/test_rcs.py | 35 +++++++++++++++++++++++++++-- 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/etc/stackalytics.conf b/etc/stackalytics.conf index 3c25c3196..1d018d3fe 100644 --- a/etc/stackalytics.conf +++ b/etc/stackalytics.conf @@ -169,6 +169,10 @@ # SSH username for gerrit review system access (string value) #ssh_username = user +# +# Proxy command to be used by ssh transport to query Gerrit behind corporate proxy +# E.g. for http proxy: "/usr/bin/ncat --proxy-type http --proxy : %s %s" +#proxy_command = # URI of translation team data (string value) #translation_team_uri = https://opendev.org/openstack/i18n/raw/branch/master/tools/zanata/translation_team.yaml diff --git a/stackalytics/processor/config.py b/stackalytics/processor/config.py index 1a697b97c..d078ef7c7 100644 --- a/stackalytics/processor/config.py +++ b/stackalytics/processor/config.py @@ -63,6 +63,8 @@ PROCESSOR_OPTS = [ help='Number of seconds to wait for remote response'), cfg.IntOpt('gerrit-retry', default=10, help='How many times to retry after Gerrit errors'), + cfg.StrOpt('proxy-command', default=None, + help='Proxy command to support Gerrit query behind proxy'), ] diff --git a/stackalytics/processor/rcs.py b/stackalytics/processor/rcs.py index 7a3d0f959..8c1796257 100644 --- a/stackalytics/processor/rcs.py +++ b/stackalytics/processor/rcs.py @@ -16,6 +16,7 @@ import json import re +from oslo_config import cfg from oslo_log import log as logging import paramiko import time @@ -28,6 +29,8 @@ PAGE_LIMIT = 100 REQUEST_COUNT_LIMIT = 20 SSH_ERRORS_LIMIT = 10 +CONF = cfg.CONF + def get_socket_tuple_from_uri(uri): stripped = re.sub(GERRIT_URI_PREFIX, '', uri) @@ -72,6 +75,11 @@ class Gerrit(Rcs): self.key_filename = None self.username = None + self.sock = None + self.proxy_command = CONF.proxy_command + if self.proxy_command: + self.proxy_command %= (self.hostname, self.port) + self.ssh_errors_limit = SSH_ERRORS_LIMIT self.client = paramiko.SSHClient() @@ -92,10 +100,12 @@ class Gerrit(Rcs): self._connect() def _connect(self): + if self.proxy_command: + self.sock = paramiko.ProxyCommand(self.proxy_command) try: self.client.connect(self.hostname, port=self.port, key_filename=self.key_filename, - username=self.username) + username=self.username, sock=self.sock) LOG.debug('Successfully connected to Gerrit') except Exception as e: LOG.error('Failed to connect to gerrit %(host)s:%(port)s. ' @@ -210,6 +220,8 @@ class Gerrit(Rcs): def close(self): self.client.close() + if self.sock: + self.sock.close() def get_rcs(uri): diff --git a/stackalytics/tests/unit/test_rcs.py b/stackalytics/tests/unit/test_rcs.py index 0dc680e7e..0777a4302 100644 --- a/stackalytics/tests/unit/test_rcs.py +++ b/stackalytics/tests/unit/test_rcs.py @@ -16,10 +16,14 @@ import json import mock +from oslo_config import cfg import testtools +from stackalytics.processor import config from stackalytics.processor import rcs +CONF = cfg.CONF + REVIEW_ONE = json.dumps( {"project": "openstack/nova", "branch": "master", "topic": "bug/1494374", "id": "Id741dfc769c02a5544691a7db49a7dbff6b11376", "number": "229382", @@ -32,6 +36,10 @@ REVIEW_END_LINE = json.dumps( class TestRcs(testtools.TestCase): + def setUp(self): + super(TestRcs, self).setUp() + CONF.register_opts(config.CONNECTION_OPTS + config.PROCESSOR_OPTS) + @mock.patch('paramiko.SSHClient') def test_setup(self, mock_client_cons): mock_client = mock.Mock() @@ -45,7 +53,30 @@ class TestRcs(testtools.TestCase): mock_connect.assert_called_once_with( 'review.openstack.org', port=rcs.DEFAULT_PORT, key_filename='key', - username='user') + username='user', sock=None) + + @mock.patch('paramiko.ProxyCommand') + @mock.patch('paramiko.SSHClient') + def test_setup_with_proxy(self, mock_client_cons, mock_proxy_command): + mock_client = mock.Mock() + mock_client_cons.return_value = mock_client + + mock_connect = mock.Mock() + mock_client.connect = mock_connect + + mock_proxy = mock.Mock() + mock_proxy_command.return_value = mock_proxy + + CONF.set_override('proxy_command', '%s:%s') + + gerrit = rcs.Gerrit('gerrit://review.openstack.org') + gerrit.setup(username='user', key_filename='key') + + mock_proxy_command.assert_called_once_with( + 'review.openstack.org:%s' % rcs.DEFAULT_PORT) + mock_connect.assert_called_once_with( + 'review.openstack.org', port=rcs.DEFAULT_PORT, key_filename='key', + username='user', sock=mock_proxy) @mock.patch('paramiko.SSHClient') def test_setup_error(self, mock_client_cons): @@ -62,7 +93,7 @@ class TestRcs(testtools.TestCase): mock_connect.assert_called_once_with( 'review.openstack.org', port=rcs.DEFAULT_PORT, key_filename='key', - username='user') + username='user', sock=None) @mock.patch('paramiko.SSHClient') @mock.patch('time.time')