diff --git a/zuul/driver/bubblewrap/__init__.py b/zuul/driver/bubblewrap/__init__.py index d2b05b1b57..9f80fd7cad 100644 --- a/zuul/driver/bubblewrap/__init__.py +++ b/zuul/driver/bubblewrap/__init__.py @@ -179,6 +179,13 @@ class BubblewrapExecutionContext(BaseExecutionContext): kwargs['gid'] = gid kwargs['uid_fd'] = passwd_r kwargs['gid_fd'] = group_r + if kwargs.get('share_net', True): + bwrap_command.append('--share-net') + if ssh_sock := kwargs.get('ssh_auth_sock', None): + bwrap_command.extend([ + '--ro-bind', ssh_sock, ssh_sock, + ]) + command = [x.format(**kwargs) for x in bwrap_command] self.log.debug("Bubblewrap command: %s", @@ -212,8 +219,7 @@ class BubblewrapDriver(Driver, WrapperInterface): # Validate basic bwrap functionality before we attempt to run # workloads under bwrap. context = self.getExecutionContext() - popen = context.getPopen(work_dir='/tmp', - ssh_auth_sock='/dev/null') + popen = context.getPopen(work_dir='/tmp') p = popen(['id'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) p.communicate() @@ -301,14 +307,12 @@ class BubblewrapDriver(Driver, WrapperInterface): '--ro-bind', '/etc/resolv.conf', '/etc/resolv.conf', '--ro-bind', '/etc/hosts', '/etc/hosts', '--ro-bind', '/etc/localtime', '/etc/localtime', - '--ro-bind', '{ssh_auth_sock}', '{ssh_auth_sock}', '--bind', '{work_dir}', '{work_dir}', '--tmpfs', '{work_dir}/tmp', '--proc', '/proc', '--dev', '/dev', '--chdir', '{work_dir}', '--unshare-all', - '--share-net', '--die-with-parent', '--uid', '{uid}', '--gid', '{gid}', @@ -361,7 +365,7 @@ def main(args=None): # The zuul-bwrap command is often run for debugging purposes. An SSH # agent may not be necessary or present in that situation. - ssh_auth_sock = os.environ.get('SSH_AUTH_SOCK', '/dev/null') + ssh_auth_sock = os.environ.get('SSH_AUTH_SOCK', None) secrets = {} if cli_args.secret: diff --git a/zuul/executor/server.py b/zuul/executor/server.py index ff99e453d4..13b1f4b2d6 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -1919,19 +1919,11 @@ class AnsibleJob(object): del data['zuul']['file_comments'] return - repo = None - for project in args['projects']: - if (project['canonical_name'] != - args['zuul']['project']['canonical_name']): - continue - repo = self.workspace_merger.getRepo(project['connection'], - project['name']) - # If the repo doesn't exist, abort - if not repo: - return - - selected_ref = args['zuul']['projects'][project['canonical_name'] - ]['checkout'] + zuul_project = args['zuul']['project'] + # TODO: set checkout on zuul.project so we don't need to look + # this up in zuul.projects + selected_ref = args['zuul']['projects'][ + zuul_project['canonical_name']]['checkout'] lines = filecomments.extractLines(fc) new_lines = {} mappers = {} @@ -1944,7 +1936,9 @@ class AnsibleJob(object): try: mapper = mappers.get(filename) if not mapper: - mapper = repo.getLineMapper(commit, selected_ref, filename) + mapper = self.getLineMapper( + zuul_project, + commit, selected_ref, filename) mappers[filename] = mapper new_lineno = mapper.mapLine(lineno) except Exception as e: @@ -1963,6 +1957,53 @@ class AnsibleJob(object): filecomments.updateLines(fc, new_lines) + def getLineMapper(self, project, commit, head, filename): + # project is args['zuul']['project'] + env_copy, popen = self.getWrappedUtilityContext() + repo_dir = os.path.join(self.jobdir.work_root, + project['src_dir'], '.git') + cmd = [ + 'git', + '--git-dir', repo_dir, + 'diff', + '--no-color', + f'{commit}..{head}', + '--', + filename, + ] + proc = popen( + cmd, + cwd=self.jobdir.work_root, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + env=env_copy, + ) + diff, err = proc.communicate() + if err: + err = err.decode('utf-8') + self.log.warning("Error from line mapping diff: %s", err.strip()) + diff = diff.decode('utf-8') + return filecomments.LineMapper(diff) + + def getWrappedUtilityContext(self): + # Get a wrapped popen suitable for running a utility program + # inside the sandbox. Does not include any mounts other than + # the work directory. + env_copy = {} + env_copy['TMP'] = self.jobdir.local_tmp + env_copy['HOME'] = self.jobdir.work_root + ro_paths = [] + rw_paths = [] + secrets = {} + wrapper = self.executor_server.execution_wrapper + context = wrapper.getExecutionContext(ro_paths, rw_paths, secrets) + return env_copy, context.getPopen( + work_dir=self.jobdir.work_root, + share_net=False, + ) + def doMergeChanges(self, items, repo_state, recent, merged_repos): try: ret = self.workspace_merger.mergeChanges( diff --git a/zuul/lib/filecomments.py b/zuul/lib/filecomments.py index fe8399a0d1..9e0501ef27 100644 --- a/zuul/lib/filecomments.py +++ b/zuul/lib/filecomments.py @@ -10,6 +10,8 @@ # License for the specific language governing permissions and limitations # under the License. +import re + import voluptuous as vs @@ -63,3 +65,49 @@ def updateLines(file_comments, lines): for key in ['start_line', 'end_line']: if key in rng: rng[key] = lines.get((path, rng[key]), rng[key]) + + +class LineMapper: + hunk_re = re.compile(r'^@@ -\d+,\d+ \+(\d+),(\d+) @@$') + + def __init__(self, diff_output): + hunk_start = None + hunk_range = None + hunk_line = None + offsets = [] + last_offset = None + for l in diff_output.split('\n'): + if len(l) == 0: + continue + m = self.hunk_re.match(l) + if m: + hunk_start = int(m.group(1)) + hunk_range = int(m.group(2)) + hunk_line = 0 + continue + if not hunk_start: + continue + if hunk_line > hunk_range: + # We have somehow run off the end of the hunk; + # shouldn't happen. + hunk_start = None + continue + if l[0] == ' ': + last_offset = None + hunk_line += 1 + continue + if not last_offset: + last_offset = [(hunk_start + hunk_line), 0] + offsets.append(last_offset) + if l[0] == '+': + last_offset[1] -= 1 + elif l[0] == '-': + last_offset[1] += 1 + self.offsets = offsets + + def mapLine(self, lineno): + new_lineno = lineno + for (start, offset) in self.offsets: + if lineno > start: + new_lineno += offset + return new_lineno diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py index b99fe4a2e4..6a188ecf36 100644 --- a/zuul/merger/merger.py +++ b/zuul/merger/merger.py @@ -69,50 +69,6 @@ class SparsePaths(enum.Enum): FULL = 1 # Checkout everything (disable) -class LineMapper: - hunk_re = re.compile(r'^@@ -\d+,\d+ \+(\d+),(\d+) @@$') - - def __init__(self, diff_output): - hunk_start = None - hunk_range = None - hunk_line = None - offsets = [] - last_offset = None - for l in diff_output.split('\n'): - m = self.hunk_re.match(l) - if m: - hunk_start = int(m.group(1)) - hunk_range = int(m.group(2)) - hunk_line = 0 - continue - if not hunk_start: - continue - if hunk_line > hunk_range: - # We have somehow run off the end of the hunk; - # shouldn't happen. - hunk_start = None - continue - if l[0] == ' ': - last_offset = None - hunk_line += 1 - continue - if not last_offset: - last_offset = [(hunk_start + hunk_line), 0] - offsets.append(last_offset) - if l[0] == '+': - last_offset[1] -= 1 - elif l[0] == '-': - last_offset[1] += 1 - self.offsets = offsets - - def mapLine(self, lineno): - new_lineno = lineno - for (start, offset) in self.offsets: - if lineno > start: - new_lineno += offset - return new_lineno - - class Repo(object): retry_attempts = 3 retry_interval = 30 @@ -1051,12 +1007,6 @@ class Repo(object): self.remote_url = None raise - def getLineMapper(self, commit, head, filename, zuul_event_id=None): - with self.createRepoObject(zuul_event_id) as repo: - diff_output = repo.git.diff( - f"{commit}..{head}", filename, no_color=True) - return LineMapper(diff_output) - def contains(self, hexsha, zuul_event_id=None): with self.createRepoObject(zuul_event_id) as repo: log = get_annotated_logger(self.log, zuul_event_id)