Merge "Perform line mapping in bubblewrap"
This commit is contained in:
Zuul
@@ -179,6 +179,13 @@ class BubblewrapExecutionContext(BaseExecutionContext):
|
||||
kwargs['gid'] = gid
|
||||
kwargs['uid_fd'] = passwd_r
|
||||
kwargs['gid_fd'] = group_r
|
||||
if kwargs.get('share_net', True):
|
||||
bwrap_command.append('--share-net')
|
||||
if ssh_sock := kwargs.get('ssh_auth_sock', None):
|
||||
bwrap_command.extend([
|
||||
'--ro-bind', ssh_sock, ssh_sock,
|
||||
])
|
||||
|
||||
command = [x.format(**kwargs) for x in bwrap_command]
|
||||
|
||||
self.log.debug("Bubblewrap command: %s",
|
||||
@@ -212,8 +219,7 @@ class BubblewrapDriver(Driver, WrapperInterface):
|
||||
# Validate basic bwrap functionality before we attempt to run
|
||||
# workloads under bwrap.
|
||||
context = self.getExecutionContext()
|
||||
popen = context.getPopen(work_dir='/tmp',
|
||||
ssh_auth_sock='/dev/null')
|
||||
popen = context.getPopen(work_dir='/tmp')
|
||||
p = popen(['id'],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
p.communicate()
|
||||
@@ -301,14 +307,12 @@ class BubblewrapDriver(Driver, WrapperInterface):
|
||||
'--ro-bind', '/etc/resolv.conf', '/etc/resolv.conf',
|
||||
'--ro-bind', '/etc/hosts', '/etc/hosts',
|
||||
'--ro-bind', '/etc/localtime', '/etc/localtime',
|
||||
'--ro-bind', '{ssh_auth_sock}', '{ssh_auth_sock}',
|
||||
'--bind', '{work_dir}', '{work_dir}',
|
||||
'--tmpfs', '{work_dir}/tmp',
|
||||
'--proc', '/proc',
|
||||
'--dev', '/dev',
|
||||
'--chdir', '{work_dir}',
|
||||
'--unshare-all',
|
||||
'--share-net',
|
||||
'--die-with-parent',
|
||||
'--uid', '{uid}',
|
||||
'--gid', '{gid}',
|
||||
@@ -361,7 +365,7 @@ def main(args=None):
|
||||
|
||||
# The zuul-bwrap command is often run for debugging purposes. An SSH
|
||||
# agent may not be necessary or present in that situation.
|
||||
ssh_auth_sock = os.environ.get('SSH_AUTH_SOCK', '/dev/null')
|
||||
ssh_auth_sock = os.environ.get('SSH_AUTH_SOCK', None)
|
||||
|
||||
secrets = {}
|
||||
if cli_args.secret:
|
||||
|
||||
+55
-14
@@ -1919,19 +1919,11 @@ class AnsibleJob(object):
|
||||
del data['zuul']['file_comments']
|
||||
return
|
||||
|
||||
repo = None
|
||||
for project in args['projects']:
|
||||
if (project['canonical_name'] !=
|
||||
args['zuul']['project']['canonical_name']):
|
||||
continue
|
||||
repo = self.workspace_merger.getRepo(project['connection'],
|
||||
project['name'])
|
||||
# If the repo doesn't exist, abort
|
||||
if not repo:
|
||||
return
|
||||
|
||||
selected_ref = args['zuul']['projects'][project['canonical_name']
|
||||
]['checkout']
|
||||
zuul_project = args['zuul']['project']
|
||||
# TODO: set checkout on zuul.project so we don't need to look
|
||||
# this up in zuul.projects
|
||||
selected_ref = args['zuul']['projects'][
|
||||
zuul_project['canonical_name']]['checkout']
|
||||
lines = filecomments.extractLines(fc)
|
||||
new_lines = {}
|
||||
mappers = {}
|
||||
@@ -1944,7 +1936,9 @@ class AnsibleJob(object):
|
||||
try:
|
||||
mapper = mappers.get(filename)
|
||||
if not mapper:
|
||||
mapper = repo.getLineMapper(commit, selected_ref, filename)
|
||||
mapper = self.getLineMapper(
|
||||
zuul_project,
|
||||
commit, selected_ref, filename)
|
||||
mappers[filename] = mapper
|
||||
new_lineno = mapper.mapLine(lineno)
|
||||
except Exception as e:
|
||||
@@ -1963,6 +1957,53 @@ class AnsibleJob(object):
|
||||
|
||||
filecomments.updateLines(fc, new_lines)
|
||||
|
||||
def getLineMapper(self, project, commit, head, filename):
|
||||
# project is args['zuul']['project']
|
||||
env_copy, popen = self.getWrappedUtilityContext()
|
||||
repo_dir = os.path.join(self.jobdir.work_root,
|
||||
project['src_dir'], '.git')
|
||||
cmd = [
|
||||
'git',
|
||||
'--git-dir', repo_dir,
|
||||
'diff',
|
||||
'--no-color',
|
||||
f'{commit}..{head}',
|
||||
'--',
|
||||
filename,
|
||||
]
|
||||
proc = popen(
|
||||
cmd,
|
||||
cwd=self.jobdir.work_root,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
start_new_session=True,
|
||||
env=env_copy,
|
||||
)
|
||||
diff, err = proc.communicate()
|
||||
if err:
|
||||
err = err.decode('utf-8')
|
||||
self.log.warning("Error from line mapping diff: %s", err.strip())
|
||||
diff = diff.decode('utf-8')
|
||||
return filecomments.LineMapper(diff)
|
||||
|
||||
def getWrappedUtilityContext(self):
|
||||
# Get a wrapped popen suitable for running a utility program
|
||||
# inside the sandbox. Does not include any mounts other than
|
||||
# the work directory.
|
||||
env_copy = {}
|
||||
env_copy['TMP'] = self.jobdir.local_tmp
|
||||
env_copy['HOME'] = self.jobdir.work_root
|
||||
ro_paths = []
|
||||
rw_paths = []
|
||||
secrets = {}
|
||||
wrapper = self.executor_server.execution_wrapper
|
||||
context = wrapper.getExecutionContext(ro_paths, rw_paths, secrets)
|
||||
return env_copy, context.getPopen(
|
||||
work_dir=self.jobdir.work_root,
|
||||
share_net=False,
|
||||
)
|
||||
|
||||
def doMergeChanges(self, items, repo_state, recent, merged_repos):
|
||||
try:
|
||||
ret = self.workspace_merger.mergeChanges(
|
||||
|
||||
@@ -10,6 +10,8 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import re
|
||||
|
||||
import voluptuous as vs
|
||||
|
||||
|
||||
@@ -63,3 +65,49 @@ def updateLines(file_comments, lines):
|
||||
for key in ['start_line', 'end_line']:
|
||||
if key in rng:
|
||||
rng[key] = lines.get((path, rng[key]), rng[key])
|
||||
|
||||
|
||||
class LineMapper:
|
||||
hunk_re = re.compile(r'^@@ -\d+,\d+ \+(\d+),(\d+) @@$')
|
||||
|
||||
def __init__(self, diff_output):
|
||||
hunk_start = None
|
||||
hunk_range = None
|
||||
hunk_line = None
|
||||
offsets = []
|
||||
last_offset = None
|
||||
for l in diff_output.split('\n'):
|
||||
if len(l) == 0:
|
||||
continue
|
||||
m = self.hunk_re.match(l)
|
||||
if m:
|
||||
hunk_start = int(m.group(1))
|
||||
hunk_range = int(m.group(2))
|
||||
hunk_line = 0
|
||||
continue
|
||||
if not hunk_start:
|
||||
continue
|
||||
if hunk_line > hunk_range:
|
||||
# We have somehow run off the end of the hunk;
|
||||
# shouldn't happen.
|
||||
hunk_start = None
|
||||
continue
|
||||
if l[0] == ' ':
|
||||
last_offset = None
|
||||
hunk_line += 1
|
||||
continue
|
||||
if not last_offset:
|
||||
last_offset = [(hunk_start + hunk_line), 0]
|
||||
offsets.append(last_offset)
|
||||
if l[0] == '+':
|
||||
last_offset[1] -= 1
|
||||
elif l[0] == '-':
|
||||
last_offset[1] += 1
|
||||
self.offsets = offsets
|
||||
|
||||
def mapLine(self, lineno):
|
||||
new_lineno = lineno
|
||||
for (start, offset) in self.offsets:
|
||||
if lineno > start:
|
||||
new_lineno += offset
|
||||
return new_lineno
|
||||
|
||||
@@ -69,50 +69,6 @@ class SparsePaths(enum.Enum):
|
||||
FULL = 1 # Checkout everything (disable)
|
||||
|
||||
|
||||
class LineMapper:
|
||||
hunk_re = re.compile(r'^@@ -\d+,\d+ \+(\d+),(\d+) @@$')
|
||||
|
||||
def __init__(self, diff_output):
|
||||
hunk_start = None
|
||||
hunk_range = None
|
||||
hunk_line = None
|
||||
offsets = []
|
||||
last_offset = None
|
||||
for l in diff_output.split('\n'):
|
||||
m = self.hunk_re.match(l)
|
||||
if m:
|
||||
hunk_start = int(m.group(1))
|
||||
hunk_range = int(m.group(2))
|
||||
hunk_line = 0
|
||||
continue
|
||||
if not hunk_start:
|
||||
continue
|
||||
if hunk_line > hunk_range:
|
||||
# We have somehow run off the end of the hunk;
|
||||
# shouldn't happen.
|
||||
hunk_start = None
|
||||
continue
|
||||
if l[0] == ' ':
|
||||
last_offset = None
|
||||
hunk_line += 1
|
||||
continue
|
||||
if not last_offset:
|
||||
last_offset = [(hunk_start + hunk_line), 0]
|
||||
offsets.append(last_offset)
|
||||
if l[0] == '+':
|
||||
last_offset[1] -= 1
|
||||
elif l[0] == '-':
|
||||
last_offset[1] += 1
|
||||
self.offsets = offsets
|
||||
|
||||
def mapLine(self, lineno):
|
||||
new_lineno = lineno
|
||||
for (start, offset) in self.offsets:
|
||||
if lineno > start:
|
||||
new_lineno += offset
|
||||
return new_lineno
|
||||
|
||||
|
||||
class Repo(object):
|
||||
retry_attempts = 3
|
||||
retry_interval = 30
|
||||
@@ -1051,12 +1007,6 @@ class Repo(object):
|
||||
self.remote_url = None
|
||||
raise
|
||||
|
||||
def getLineMapper(self, commit, head, filename, zuul_event_id=None):
|
||||
with self.createRepoObject(zuul_event_id) as repo:
|
||||
diff_output = repo.git.diff(
|
||||
f"{commit}..{head}", filename, no_color=True)
|
||||
return LineMapper(diff_output)
|
||||
|
||||
def contains(self, hexsha, zuul_event_id=None):
|
||||
with self.createRepoObject(zuul_event_id) as repo:
|
||||
log = get_annotated_logger(self.log, zuul_event_id)
|
||||
|
||||
Reference in New Issue
Block a user