From edadfed2568f48fbe1be4b6c5560075e08340a07 Mon Sep 17 00:00:00 2001 From: Paul Belanger Date: Thu, 5 Oct 2017 16:04:27 -0400 Subject: [PATCH] Create git_http_low_speed_limit / git_http_low_speed_time Expose 2 new settings to zuul.conf allowing an operator better control over them. By default we set the speed limit to 1000 bytes and speed time to 30 seconds. Change-Id: I9da80fcfc312cbc12ea11ee7284eaec23adb97c9 Signed-off-by: Paul Belanger --- doc/source/admin/components.rst | 16 ++++++++++++++++ tests/unit/test_merger_repo.py | 4 ++-- zuul/executor/server.py | 10 +++++++--- zuul/merger/merger.py | 18 ++++++++++-------- zuul/merger/server.py | 10 +++++++--- 5 files changed, 42 insertions(+), 16 deletions(-) diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst index 5e7e0e1a22..464cb602f7 100644 --- a/doc/source/admin/components.rst +++ b/doc/source/admin/components.rst @@ -265,6 +265,22 @@ The following section of ``zuul.conf`` is used by the merger: Directory in which Zuul should clone git repositories. + .. attr:: git_http_low_speed_limit + :default: 1000 + + If the HTTP transfer speed is less then git_http_low_speed_limit for + longer then git_http_low_speed_time, the transfer is aborted. + + Value in bytes, setting to 0 will disable. + + .. attr:: git_http_low_speed_time + :default: 30 + + If the HTTP transfer speed is less then git_http_low_speed_limit for + longer then git_http_low_speed_time, the transfer is aborted. + + Value in seconds, setting to 0 will disable. + .. attr:: git_user_email Value to pass to `git config user.email diff --git a/tests/unit/test_merger_repo.py b/tests/unit/test_merger_repo.py index 8aafabf65f..67be2730b7 100644 --- a/tests/unit/test_merger_repo.py +++ b/tests/unit/test_merger_repo.py @@ -49,7 +49,7 @@ class TestMergerRepo(ZuulTestCase): msg='.git file in submodule should be a file') work_repo = Repo(parent_path, self.workspace_root, - 'none@example.org', 'User Name') + 'none@example.org', 'User Name', '0', '0') self.assertTrue( os.path.isdir(os.path.join(self.workspace_root, 'subdir')), msg='Cloned repository has a submodule placeholder directory') @@ -60,7 +60,7 @@ class TestMergerRepo(ZuulTestCase): sub_repo = Repo( os.path.join(self.upstream_root, 'org/project2'), os.path.join(self.workspace_root, 'subdir'), - 'none@example.org', 'User Name') + 'none@example.org', 'User Name', '0', '0') self.assertTrue(os.path.exists( os.path.join(self.workspace_root, 'subdir', '.git')), msg='Cloned over the submodule placeholder') diff --git a/zuul/executor/server.py b/zuul/executor/server.py index 82921fb641..52be5a38e7 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -562,6 +562,10 @@ class ExecutorServer(object): 'disk_limit_per_job', 250)) self.merge_email = get_default(self.config, 'merger', 'git_user_email') self.merge_name = get_default(self.config, 'merger', 'git_user_name') + self.merge_speed_limit = get_default( + config, 'merger', 'git_http_low_speed_limit', '1000') + self.merge_speed_time = get_default( + config, 'merger', 'git_http_low_speed_time', '30') execution_wrapper_name = get_default(self.config, 'executor', 'execution_wrapper', 'bubblewrap') load_multiplier = float(get_default(self.config, 'executor', @@ -618,9 +622,9 @@ class ExecutorServer(object): cache_root = self.merge_root else: cache_root = None - return zuul.merger.merger.Merger(root, self.connections, - self.merge_email, self.merge_name, - cache_root, logger) + return zuul.merger.merger.Merger( + root, self.connections, self.merge_email, self.merge_name, + self.merge_speed_limit, self.merge_speed_time, cache_root, logger) def start(self): self._running = True diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py index 7b732c7428..fbacbeeced 100644 --- a/zuul/merger/merger.py +++ b/zuul/merger/merger.py @@ -44,16 +44,15 @@ class ZuulReference(git.Reference): class Repo(object): - def __init__(self, remote, local, email, username, sshkey=None, - cache_path=None, logger=None): + def __init__(self, remote, local, email, username, speed_limit, speed_time, + sshkey=None, cache_path=None, logger=None): if logger is None: self.log = logging.getLogger("zuul.Repo") else: self.log = logger - # TODO(pabelanger): Expose to user via zuul.conf. self.env = { - 'GIT_HTTP_LOW_SPEED_LIMIT': '1000', - 'GIT_HTTP_LOW_SPEED_TIME': '30', + 'GIT_HTTP_LOW_SPEED_LIMIT': speed_limit, + 'GIT_HTTP_LOW_SPEED_TIME': speed_time, } if sshkey: self.env['GIT_SSH_COMMAND'] = 'ssh -i %s' % (sshkey,) @@ -297,7 +296,7 @@ class Repo(object): class Merger(object): def __init__(self, working_root, connections, email, username, - cache_root=None, logger=None): + speed_limit, speed_time, cache_root=None, logger=None): self.logger = logger if logger is None: self.log = logging.getLogger("zuul.Merger") @@ -310,6 +309,8 @@ class Merger(object): self.connections = connections self.email = email self.username = username + self.speed_limit = speed_limit + self.speed_time = speed_time self.cache_root = cache_root def _addProject(self, hostname, project_name, url, sshkey): @@ -322,8 +323,9 @@ class Merger(object): project_name) else: cache_path = None - repo = Repo(url, path, self.email, self.username, - sshkey, cache_path, self.logger) + repo = Repo( + url, path, self.email, self.username, self.speed_limit, + self.speed_time, sshkey, cache_path, self.logger) self.repos[key] = repo except Exception: diff --git a/zuul/merger/server.py b/zuul/merger/server.py index 881209d65b..765d9e0ba9 100644 --- a/zuul/merger/server.py +++ b/zuul/merger/server.py @@ -33,9 +33,13 @@ class MergeServer(object): '/var/lib/zuul/merger-git') merge_email = get_default(self.config, 'merger', 'git_user_email') merge_name = get_default(self.config, 'merger', 'git_user_name') - - self.merger = merger.Merger(merge_root, connections, merge_email, - merge_name) + speed_limit = get_default( + config, 'merger', 'git_http_low_speed_limit', '1000') + speed_time = get_default( + config, 'merger', 'git_http_low_speed_time', '30') + self.merger = merger.Merger( + merge_root, connections, merge_email, merge_name, speed_limit, + speed_time) def start(self): self._running = True