From f1e7ffab934b7522111d1ec2e5e0e8ffe3c29edc Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Wed, 17 Feb 2021 09:22:46 -0800 Subject: [PATCH] Remove zuul-migrate This was intended as a one-time helper program to help people upgrade from Zuul v2 to v3. It did not cover all use cases, and has not been kept up to date or improved. It's time to remove it before the v4 release. Change-Id: I12cdcedb5baabd8fa0937a6ea21590259093ead1 --- .zuul.yaml | 12 - playbooks/zuul-migrate/post.yaml | 26 - playbooks/zuul-migrate/run.yaml | 10 - setup.cfg | 3 - zuul/cmd/migrate.py | 1744 ------------------------------ 5 files changed, 1795 deletions(-) delete mode 100644 playbooks/zuul-migrate/post.yaml delete mode 100644 playbooks/zuul-migrate/run.yaml delete mode 100644 zuul/cmd/migrate.py diff --git a/.zuul.yaml b/.zuul.yaml index e35aa35944..711cb69c3f 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -224,9 +224,6 @@ vars: tox_install_bindep: false - tox-py36: - irrelevant-files: - - zuul/cmd/migrate.py - - playbooks/zuul-migrate/.* nodeset: ubuntu-bionic timeout: 4800 # 80 minutes vars: &zuul_tox_vars @@ -238,9 +235,6 @@ YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs" post-run: playbooks/common/post-system-logs.yaml - tox-py38: - irrelevant-files: - - zuul/cmd/migrate.py - - playbooks/zuul-migrate/.* timeout: 4800 # 80 minutes nodeset: ubuntu-bionic vars: *zuul_tox_vars @@ -276,16 +270,10 @@ vars: tox_install_bindep: false - tox-py36: - irrelevant-files: - - zuul/cmd/migrate.py - - playbooks/zuul-migrate/.* nodeset: ubuntu-bionic timeout: 4800 # 80 minutes vars: *zuul_tox_vars - tox-py38: - irrelevant-files: - - zuul/cmd/migrate.py - - playbooks/zuul-migrate/.* timeout: 4800 # 80 minutes nodeset: ubuntu-bionic vars: *zuul_tox_vars diff --git a/playbooks/zuul-migrate/post.yaml b/playbooks/zuul-migrate/post.yaml deleted file mode 100644 index 3ad16bc918..0000000000 --- a/playbooks/zuul-migrate/post.yaml +++ /dev/null @@ -1,26 +0,0 @@ -- hosts: all - tasks: - - - name: Collect openstack-zuul-jobs generated job config - synchronize: - dest: "{{ zuul.executor.log_root }}/openstack-zuul-jobs" - mode: pull - src: "src/opendev.org/openstack/openstack-zuul-jobs/zuul.d" - verify_host: true - no_log: true - - - name: Collect project generated job config - synchronize: - dest: "{{ zuul.executor.log_root }}/openstack-zuul-jobs" - mode: pull - src: "src/opendev.org/openstack/project-config/zuul.d" - verify_host: true - no_log: true - - - name: Collect generated playbooks - synchronize: - dest: "{{ zuul.executor.log_root }}/playbooks" - mode: pull - src: "src/opendev.org/openstack/openstack-zuul-jobs/playbooks/legacy" - verify_host: true - no_log: true diff --git a/playbooks/zuul-migrate/run.yaml b/playbooks/zuul-migrate/run.yaml deleted file mode 100644 index 562dc12356..0000000000 --- a/playbooks/zuul-migrate/run.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- hosts: all - tasks: - - - name: Install migration dependencies - command: "python3 -m pip install --user src/opendev.org/zuul/zuul[migrate]" - - - name: Migrate the data - command: tools/run-migration.sh -v --final - args: - chdir: src/opendev.org/zuul/zuul diff --git a/setup.cfg b/setup.cfg index 20fe7b39b6..c404fdc052 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,7 +33,6 @@ console_scripts = zuul-executor = zuul.cmd.executor:main zuul-bwrap = zuul.driver.bubblewrap:main zuul-web = zuul.cmd.web:main - zuul-migrate = zuul.cmd.migrate:main zuul-fingergw = zuul.cmd.fingergw:main zuul-manage-ansible = zuul.cmd.manage_ansible:main @@ -46,8 +45,6 @@ warning-is-error = 1 [extras] mysql_reporter= PyMySQL -migrate= - jenkins-job-builder==1.6.2 # The container image build process will look at the python extras and install # any that match the target container name. It makes a container image for # every console_script listed above. The extra name must have underscores. diff --git a/zuul/cmd/migrate.py b/zuul/cmd/migrate.py deleted file mode 100644 index 7fe1d12a96..0000000000 --- a/zuul/cmd/migrate.py +++ /dev/null @@ -1,1744 +0,0 @@ -# Copyright 2017 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# TODO(mordred): -# * Read and apply filters from the jobs: section -# * Figure out shared job queues -# * Emit job definitions -# * figure out from builders whether or not it's a normal job or a -# a devstack-legacy job -# * Handle emitting arbitrary tox jobs (see tox-py27dj18) - -import argparse -import collections -import copy -import itertools -import getopt -import logging -import os -import subprocess -import tempfile -import re -from typing import Any, Dict, List, Optional - -import jenkins_jobs.builder -from jenkins_jobs.formatter import deep_format -import jenkins_jobs.formatter -from jenkins_jobs.parser import matches -import jenkins_jobs.parser -import yaml - -JOB_MATCHERS = {} # type: Dict[str, Dict[str, Dict]] -TEMPLATES_TO_EXPAND = {} # type: Dict[str, List] -JOBS_FOR_EXPAND = collections.defaultdict(dict) # type: ignore -JOBS_BY_ORIG_TEMPLATE = {} # type: ignore -SUFFIXES = [] # type: ignore -SKIP_MACROS = [] # type: ignore -ENVIRONMENT = '{{ zuul | zuul_legacy_vars }}' -DESCRIPTION = """Migrate zuul v2 and Jenkins Job Builder to Zuul v3. - -This program takes a zuul v2 layout.yaml and a collection of Jenkins Job -Builder job definitions and transforms them into a Zuul v3 config. An -optional mapping config can be given that defines how to map old jobs -to new jobs. -""" - - -def deal_with_shebang(data): - # Ansible shell blocks do not honor shebang lines. That's fine - but - # we do have a bunch of scripts that have either nothing, -x, -xe, - # -ex or -eux. Transform those into leading set commands - if not data.startswith('#!'): - return (None, data) - data_lines = data.split('\n') - data_lines.reverse() - shebang = data_lines.pop() - split_line = shebang.split() - # Strip the # and the ! - executable = split_line[0][2:] - if executable == '/bin/sh': - # Ansible default - executable = None - if len(split_line) > 1: - flag_x = False - flag_e = False - flag_u = False - optlist, args = getopt.getopt(split_line[1:], 'uex') - for opt, _ in optlist: - if opt == '-x': - flag_x = True - elif opt == '-e': - flag_e = True - elif opt == '-u': - flag_u = True - - if flag_x: - data_lines.append('set -x') - if flag_e: - data_lines.append('set -e') - if flag_u: - data_lines.append('set -u') - data_lines.reverse() - data = '\n'.join(data_lines).lstrip() - return (executable, data) - - -def _extract_from_vars(line): - # export PROJECTS="openstack/blazar $PROJECTS" - # export DEVSTACK_PROJECT_FROM_GIT=python-swiftclient - # export DEVSTACK_PROJECT_FROM_GIT="python-octaviaclient" - # export DEVSTACK_PROJECT_FROM_GIT+=",glean" - projects = [] - line = line.replace('"', '').replace('+', '').replace(',', ' ') - if (line.startswith('export PROJECTS') or - line.startswith('export DEVSTACK_PROJECT_FROM_GIT')): - nothing, project_string = line.split('=') - project_string = project_string.replace('$PROJECTS', '').strip() - projects = project_string.split() - return projects - - -def extract_projects(data): - # clonemap: - # - name: openstack/windmill - # dest: . - # EOF - projects = [] - data_lines = data.split('\n') - in_clonemap = False - in_clonemap_cli = False - for line in data_lines: - line = line.strip() - if line == 'clonemap:': - in_clonemap = True - continue - elif line == 'EOF': - in_clonemap = False - continue - elif line.startswith('/usr/zuul-env/bin/zuul-cloner'): - in_clonemap_cli = True - continue - elif in_clonemap_cli and not line.endswith('\\'): - in_clonemap_cli = False - continue - if in_clonemap: - if line.startswith('- name:'): - garbage, project = line.split(':') - project = project.strip().replace("'", '').replace('"', '') - if project == '$ZUUL_PROJECT': - continue - projects.append(project) - elif in_clonemap_cli and line.startswith('openstack/'): - line = line.replace('\\', '').strip() - projects.append(line) - elif in_clonemap_cli: - continue - else: - projects.extend(_extract_from_vars(line)) - return projects - - -def expand_project_names(required, full): - projects = [] - for name in full: - repo = name - if '/' in name: - org, repo = name.split('/') - if repo in required or name in required: - projects.append(name) - return projects - - -def list_to_project_dicts(projects): - project_dicts = dict() - for project in projects: - project_dicts[project['project']['name']] = project['project'] - return project_dicts - - -def project_dicts_to_list(project_dicts): - project_config = [] - for key in sorted(project_dicts.keys()): - project_config.append(dict(project=project_dicts[key])) - return project_config - - -def merge_project_dict(project_dicts, name, project): - if name not in project_dicts: - project_dicts[name] = project - return - - old = project_dicts[name] - for key in project: - if key not in old: - old[key] = project[key] - elif isinstance(old[key], list): - old[key].extend(project[key]) - elif isinstance(old[key], dict) and 'jobs' in old[key]: - old[key]['jobs'].extend(project[key]['jobs']) - return - - -def normalize_project_expansions(): - remove_from_job_matchers = [] - # First find the matchers that are the same for all jobs - for job_name, project in copy.deepcopy(JOBS_FOR_EXPAND).items(): - JOB_MATCHERS[job_name] = None - for project_name, expansion in project.items(): - if not JOB_MATCHERS[job_name]: - JOB_MATCHERS[job_name] = copy.deepcopy(expansion['info']) - else: - if JOB_MATCHERS[job_name] != expansion['info']: - # We have different expansions for this job, it can't be - # done at the job level - remove_from_job_matchers.append(job_name) - - for job_name in remove_from_job_matchers: - JOB_MATCHERS.pop(job_name, None) - - # Second, find out which projects need to expand a given template - for job_name, project in copy.deepcopy(JOBS_FOR_EXPAND).items(): - # There is a job-level expansion for this one - if job_name in JOB_MATCHERS: - continue - for project_name, expansion in project.items(): - TEMPLATES_TO_EXPAND[project_name] = [] - if expansion['info']: - # There is an expansion for this project - TEMPLATES_TO_EXPAND[project_name].append(expansion['template']) - - -# from : -# http://stackoverflow.com/questions/8640959/how-can-i-control-what-scalar-form-pyyaml-uses-for-my-data # noqa -def should_use_block(value): - for c in u"\u000a\u000d\u001c\u001d\u001e\u0085\u2028\u2029": - if c in value: - return True - return False - - -def my_represent_scalar(self, tag, value, style=None): - if style is None: - if should_use_block(value): - style = '|' - else: - style = self.default_style - - node = yaml.representer.ScalarNode(tag, value, style=style) - if self.alias_key is not None: - self.represented_objects[self.alias_key] = node - return node - - -def project_representer(dumper, data): - return dumper.represent_mapping('tag:yaml.org,2002:map', - data.items()) - - -def construct_yaml_map(self, node): - data = collections.OrderedDict() - yield data - value = self.construct_mapping(node) - - if isinstance(node, yaml.MappingNode): - self.flatten_mapping(node) - else: - raise yaml.constructor.ConstructorError( - None, None, - 'expected a mapping node, but found %s' % node.id, - node.start_mark) - - mapping = collections.OrderedDict() - for key_node, value_node in node.value: - key = self.construct_object(key_node, deep=False) - try: - hash(key) - except TypeError as exc: - raise yaml.constructor.ConstructorError( - 'while constructing a mapping', node.start_mark, - 'found unacceptable key (%s)' % exc, key_node.start_mark) - value = self.construct_object(value_node, deep=False) - mapping[key] = value - data.update(mapping) - - -class IndentedEmitter(yaml.emitter.Emitter): - def expect_block_sequence(self): - self.increase_indent(flow=False, indentless=False) - self.state = self.expect_first_block_sequence_item - - -class IndentedDumper(IndentedEmitter, yaml.serializer.Serializer, - yaml.representer.Representer, yaml.resolver.Resolver): - def __init__(self, stream, - default_style=None, default_flow_style=None, - canonical=None, indent=None, width=None, - allow_unicode=None, line_break=None, - encoding=None, explicit_start=None, explicit_end=None, - version=None, tags=None): - IndentedEmitter.__init__( - self, stream, canonical=canonical, - indent=indent, width=width, - allow_unicode=allow_unicode, - line_break=line_break) - yaml.serializer.Serializer.__init__( - self, encoding=encoding, - explicit_start=explicit_start, - explicit_end=explicit_end, - version=version, tags=tags) - yaml.representer.Representer.__init__( - self, default_style=default_style, - default_flow_style=default_flow_style) - yaml.resolver.Resolver.__init__(self) - - -def ordered_load(stream, *args, **kwargs): - return yaml.load(stream=stream, *args, **kwargs) - - -def ordered_dump(data, stream=None, *args, **kwargs): - dumper = IndentedDumper - # We need to do this because of how template expasion into a project - # works. Without it, we end up with YAML references to the expanded jobs. - dumper.ignore_aliases = lambda self, data: True - - output = yaml.dump( - data, default_flow_style=False, - Dumper=dumper, width=80, *args, **kwargs).replace( - '\n -', '\n\n -') - if stream: - stream.write(output) - else: - return output - - -def get_single_key(var): - if isinstance(var, str): - return var - elif isinstance(var, list): - return var[0] - return list(var.keys())[0] - - -def has_single_key(var): - if isinstance(var, list): - return len(var) == 1 - if isinstance(var, str): - return True - dict_keys = list(var.keys()) - if len(dict_keys) != 1: - return False - if var[get_single_key(var)]: - return False - return True - - -def combination_matches(combination, match_combinations): - """ - Checks if the given combination is matches for any of the given combination - globs, being those a set of combinations where if a key is missing, it's - considered matching - - (key1=2, key2=3) - - would match the combination match: - (key2=3) - - but not: - (key1=2, key2=2) - """ - for cmatch in match_combinations: - for key, val in combination.items(): - if cmatch.get(key, val) != val: - break - else: - return True - return False - - -def expandYamlForTemplateJob(self, project, template, jobs_glob=None): - dimensions = [] - template_name = template['name'] - orig_template = copy.deepcopy(template) - - # reject keys that are not useful during yaml expansion - for k in ['jobs']: - project.pop(k) - excludes = project.pop('exclude', []) - for (k, v) in project.items(): - tmpk = '{{{0}}}'.format(k) - if tmpk not in template_name: - continue - if type(v) == list: - dimensions.append(zip([k] * len(v), v)) - # XXX somewhat hackish to ensure we actually have a single - # pass through the loop - if len(dimensions) == 0: - dimensions = [(("", ""),)] - - for values in itertools.product(*dimensions): - params = copy.deepcopy(project) - params = self.applyDefaults(params, template) - - expanded_values = {} - for (k, v) in values: - if isinstance(v, dict): - inner_key = next(iter(v)) - expanded_values[k] = inner_key - expanded_values.update(v[inner_key]) - else: - expanded_values[k] = v - - params.update(expanded_values) - params = deep_format(params, params) - if combination_matches(params, excludes): - log = logging.getLogger("zuul.Migrate.YamlParser") - log.debug('Excluding combination %s', str(params)) - continue - - allow_empty_variables = self.config \ - and self.config.has_section('job_builder') \ - and self.config.has_option( - 'job_builder', 'allow_empty_variables') \ - and self.config.getboolean( - 'job_builder', 'allow_empty_variables') - - for key in template: - if key not in params: - params[key] = template[key] - - params['template-name'] = template_name - project_name = params['name'] - params['name'] = '$ZUUL_SHORT_PROJECT_NAME' - expanded = deep_format(template, params, allow_empty_variables) - - job_name = expanded.get('name') - templated_job_name = job_name - if job_name: - job_name = job_name.replace( - '$ZUUL_SHORT_PROJECT_NAME', project_name) - expanded['name'] = job_name - if jobs_glob and not matches(job_name, jobs_glob): - continue - - self.formatDescription(expanded) - expanded['orig_template'] = orig_template - expanded['template_name'] = template_name - self.jobs.append(expanded) - JOBS_BY_ORIG_TEMPLATE[templated_job_name] = expanded - - -jenkins_jobs.parser.YamlParser.expandYamlForTemplateJob = \ - expandYamlForTemplateJob - - -class JJB(jenkins_jobs.builder.Builder): - def __init__(self): - self.global_config = None - self._plugins_list = [] - - def expandComponent(self, component_type, component, template_data): - component_list_type = component_type + 's' - new_components = [] - if isinstance(component, dict): - name, component_data = next(iter(component.items())) - if template_data: - component_data = jenkins_jobs.formatter.deep_format( - component_data, template_data, True) - else: - name = component - component_data = {} - - new_component = self.parser.data.get(component_type, {}).get(name) - if new_component: - for new_sub_component in new_component[component_list_type]: - new_components.extend( - self.expandComponent(component_type, - new_sub_component, component_data)) - else: - new_components.append({name: component_data}) - return new_components - - def expandMacros(self, job): - for component_type in ['builder', 'publisher', 'wrapper']: - component_list_type = component_type + 's' - new_components = [] - for new_component in job.get(component_list_type, []): - # Skip macros defined in mappings.yaml - if new_component in SKIP_MACROS: - continue - new_components.extend(self.expandComponent(component_type, - new_component, {})) - job[component_list_type] = new_components - - -class OldProject: - def __init__(self, name, gate_jobs): - self.name = name - self.gate_jobs = gate_jobs - - -class OldJob: - def __init__(self, name): - self.name = name - self.queue_name = None - - def __repr__(self): - return self.name - - -class Job: - - log = logging.getLogger("zuul.Migrate") - - def __init__(self, - orig: str, - name: str=None, - content: Dict[str, Any]=None, - vars: Dict[str, str]=None, - nodes: List[str]=None, - parent=None) -> None: - self.orig = orig - self.voting = True - self.name = name - self.content = content.copy() if content else None - self.vars = vars or {} - self.required_projects = [] # type: ignore - self.nodes = nodes or [] - self.parent = parent - self.branch = None - self.files = None - self.jjb_job = None - self.emit = True - - if self.content and not self.name: - self.name = get_single_key(content) - if not self.name: - self.name = self.orig - self.name = self.name.replace('-{name}', '').replace('{name}-', '') - - for suffix in SUFFIXES: - suffix = '-{suffix}'.format(suffix=suffix) - - if self.name.endswith(suffix): - self.name = self.name.replace(suffix, '') - - def _stripNodeName(self, node): - node_key = '-{node}'.format(node=node) - self.name = self.name.replace(node_key, '') - - def setNoEmit(self): - self.emit = False - - def setVars(self, vars): - self.vars = vars - - def setParent(self, parent): - self.parent = parent - - def extractNode(self, default_node, labels): - matching_label = None - for label in labels: - if label in self.orig: - if not matching_label: - matching_label = label - elif len(label) > len(matching_label): - matching_label = label - - if matching_label: - if matching_label == default_node: - self._stripNodeName(matching_label) - else: - self.nodes.append(matching_label) - - def getDepends(self): - return [self.parent.name] - - def getNodes(self): - return self.nodes - - def addJJBJob(self, jobs): - if '{name}' in self.orig: - self.jjb_job = JOBS_BY_ORIG_TEMPLATE[self.orig.format( - name='$ZUUL_SHORT_PROJECT_NAME')] - else: - self.jjb_job = jobs[self.orig] - - def getTimeout(self): - timeout = None - if self.jjb_job: - for wrapper in self.jjb_job.get('wrappers', []): - if isinstance(wrapper, dict): - build_timeout = wrapper.get( - 'build-timeout', wrapper.get('timeout')) - if isinstance(build_timeout, dict): - timeout = build_timeout.get('timeout') - if timeout is not None: - timeout = int(timeout) * 60 - # NOTE: This should be read from tenant config - if timeout > 10800: - timeout = 10800 - - return timeout - - @property - def short_name(self): - return self.name.replace('legacy-', '') - - @property - def job_path(self): - return 'playbooks/legacy/{name}'.format(name=self.short_name) - - def _getRsyncOptions(self, source): - # If the source starts with ** then we want to match any - # number of directories, so don't anchor the include filter. - # If it does not start with **, then the intent is likely to - # at least start by matching an immediate file or subdirectory - # (even if later we have a ** in the middle), so in this case, - # anchor it to the root of the transfer (the workspace). - if not source.startswith('**'): - source = os.path.join('/', source) - # These options mean: include the thing we want, include any - # directories (so that we continue to search for the thing we - # want no matter how deep it is), exclude anything that - # doesn't match the thing we want or is a directory, then get - # rid of empty directories left over at the end. - rsync_opts = ['--include=%s' % source, - '--include=*/', - '--exclude=*', - '--prune-empty-dirs'] - return rsync_opts - - def _makeSCPTask(self, publisher): - # NOTE(mordred) About docs-draft manipulation: - # The target of html/ was chosen to put the node contents into the - # html dir inside of logs such that if the node's contents have an - # index.html in them setting the success-url to html/ will render - # things as expected. Existing builder macros look like: - # - # - publisher: - # name: upload-sphinx-draft - # publishers: - # - scp: - # site: 'static.openstack.org' - # files: - # - target: 'docs-draft/$LOG_PATH' - # source: 'doc/build/html/**' - # keep-hierarchy: true - # copy-after-failure: true - # - # Which is pulling the tree of the remote html directory starting with - # doc/build/html and putting that whole thing into - # docs-draft/$LOG_PATH. - # - # Then there is a success-pattern in layout.yaml that looks like: - # - # http://{url}/{log_path}/doc/build/html/ - # - # Which gets reports. There are many variations on that URL. So rather - # than needing to figure out varying success-urls to report in v3, - # we'll remote the ** and not process this through the rsync_opts - # processing we use for the other publishers, but instead will just - # pass doc/build/html/ to get the contents of doc/build/html/ and we'll - # put those in {{ log_root }}/html/ locally meaning the success-url - # can always be html/. This should work for all values of source - # from v2. - tasks = [] - artifacts = False - draft = False - site = publisher['scp']['site'] - for scpfile in publisher['scp']['files']: - if 'ZUUL_PROJECT' in scpfile.get('source', ''): - self.log.error( - "Job {name} uses ZUUL_PROJECT in source".format( - name=self.name)) - continue - - if scpfile.get('copy-console'): - continue - else: - src = "{{ ansible_user_dir }}/workspace/" - rsync_opts = self._getRsyncOptions(scpfile['source']) - - target = scpfile['target'] - # TODO(mordred) Generalize this next section, it's SUPER - # openstack specific. We can likely do this in mapping.yaml - if site == 'static.openstack.org': - for f in ('service-types', 'specs'): - if target.startswith(f): - self.log.error( - "Job {name} uses {f} publishing".format( - name=self.name, f=f)) - continue - if target.startswith('docs-draft'): - target = "{{ zuul.executor.log_root }}/html/" - src = scpfile['source'].replace('**', '') - rsync_opts = None - draft = True - else: - target = target.replace( - 'logs/$LOG_PATH', - "{{ zuul.executor.log_root }}") - elif site == 'tarballs.openstack.org': - if not target.startswith('tarballs'): - self.log.error( - 'Job {name} wants to publish artifacts to non' - ' tarballs dir'.format(name=self.name)) - continue - if target.startswith('tarballs/ci'): - target = target.split('/', 3)[-1] - else: - target = target.split('/', 2)[-1] - target = "{{ zuul.executor.work_root }}/artifacts/" + target - artifacts = True - elif site == 'yaml2ical': - self.log.error('Job {name} uses yaml2ical publisher') - continue - - syncargs = collections.OrderedDict() - syncargs['src'] = src - syncargs['dest'] = target - syncargs['mode'] = 'pull' - syncargs['copy_links'] = True - syncargs['verify_host'] = True - if rsync_opts: - syncargs['rsync_opts'] = rsync_opts - task = collections.OrderedDict() - task['name'] = 'Copy files from {src} on node'.format(src=src) - task['synchronize'] = syncargs - # We don't use retry_args here because there is a bug in - # the synchronize module that breaks subsequent attempts at - # retrying. Better to try once and get an accurate error - # message if it fails. - # https://github.com/ansible/ansible/issues/18281 - tasks.append(task) - - if artifacts: - ensure_task = collections.OrderedDict() - ensure_task['name'] = 'Ensure artifacts directory exists' - ensure_task['file'] = collections.OrderedDict() - ensure_task['file']['path'] = \ - "{{ zuul.executor.work_root }}/artifacts" - ensure_task['file']['state'] = 'directory' - ensure_task['delegate_to'] = 'localhost' - tasks.insert(0, ensure_task) - return dict(tasks=tasks, artifacts=artifacts, draft=draft) - - def _emitShellTask(self, data, syntax_check): - shell, data = deal_with_shebang(data) - task = collections.OrderedDict() - # Putting the data directly into shell: causes here docs to break. - task['shell'] = collections.OrderedDict() - task['shell']['cmd'] = data - if shell: - task['shell']['executable'] = shell - task['shell']['chdir'] = '{{ ansible_user_dir }}/workspace' - - if syntax_check: - # Emit a test playbook with this shell task in it then run - # ansible-playbook --syntax-check on it. This will fail if there - # are embedding issues, such as with unbalanced single quotes - # The end result should be less scripts and more shell - play = dict(hosts='all', tasks=[task]) - (fd, tmp_path) = tempfile.mkstemp() - try: - f = os.fdopen(fd, 'w') - ordered_dump([play], f) - f.close() - proc = subprocess.run( - ['ansible-playbook', '--syntax-check', tmp_path], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - if proc.returncode != 0: - # Return of None means we must emit a script - self.log.error( - "Job {name} had an ansible syntax error, using script" - " instead of shell".format(name=self.name)) - return None - finally: - os.unlink(tmp_path) - return task - - def _emitScriptContent(self, data, playbook_dir, seq): - script_fn = '%s-%02d.sh' % (self.short_name, seq) - script_path = os.path.join(playbook_dir, script_fn) - - with open(script_path, 'w') as script: - if not data.startswith('#!'): - data = '#!/bin/bash -x\n %s' % (data,) - script.write(data) - - task = collections.OrderedDict() - task['name'] = 'Running playbooks/legacy/{playbook}'.format( - playbook=script_fn) - task['script'] = script_fn - return task - - def _makeBuilderTask(self, playbook_dir, builder, sequence, syntax_check): - # Don't write a script to echo the template line - # TODO(mordred) Put these into mapping.yaml - if builder['shell'].startswith('echo JJB template: '): - return - if 'echo "Detailed logs:' in builder['shell']: - return - if ('cat /etc/dib-builddate.txt' in builder['shell'] and - 'echo "Network configuration data..."' in builder['shell']): - return - - task = self._emitShellTask(builder['shell'], syntax_check) - if not task: - task = self._emitScriptContent( - builder['shell'], playbook_dir, sequence) - task['environment'] = ENVIRONMENT - - return task - - def _transformPublishers(self, jjb_job): - early_publishers = [] - late_publishers = [] - old_publishers = jjb_job.get('publishers', []) - for publisher in old_publishers: - early_scpfiles = [] - late_scpfiles = [] - if 'scp' not in publisher: - early_publishers.append(publisher) - continue - copy_console = False - for scpfile in publisher['scp']['files']: - if scpfile.get('copy-console'): - scpfile['keep-hierarchy'] = True - late_scpfiles.append(scpfile) - copy_console = True - else: - early_scpfiles.append(scpfile) - publisher['scp']['files'] = early_scpfiles + late_scpfiles - if copy_console: - late_publishers.append(publisher) - else: - early_publishers.append(publisher) - publishers = early_publishers + late_publishers - if old_publishers != publishers: - self.log.debug("Transformed job publishers") - return early_publishers, late_publishers - - def emitPlaybooks(self, jobsdir, syntax_check=False): - has_artifacts = False - has_draft = False - if not self.jjb_job: - if self.emit: - self.log.error( - 'Job {name} has no job content'.format(name=self.name)) - return False, False, False - - playbook_dir = os.path.join(jobsdir, self.job_path) - if not os.path.exists(playbook_dir): - os.makedirs(playbook_dir) - - run_playbook = os.path.join(self.job_path, 'run.yaml') - post_playbook = os.path.join(self.job_path, 'post.yaml') - - tasks = [] - workspace_task = collections.OrderedDict() - workspace_task['name'] = "Ensure legacy workspace directory" - workspace_task['file'] = collections.OrderedDict() - workspace_task['file']['path'] = '{{ ansible_user_dir }}/workspace' - workspace_task['file']['state'] = 'directory' - tasks.append(workspace_task) - - sequence = 0 - for builder in self.jjb_job.get('builders', []): - if 'shell' in builder: - self.required_projects.extend( - extract_projects(builder['shell'])) - task = self._makeBuilderTask( - playbook_dir, builder, sequence, syntax_check) - if task: - if 'script' in task: - sequence += 1 - tasks.append(task) - play = collections.OrderedDict() - play['hosts'] = 'all' - play['name'] = 'Autoconverted job {name} from old job {old}'.format( - name=self.name, old=self.orig) - play['tasks'] = tasks - - with open(run_playbook, 'w') as run_playbook_out: - ordered_dump([play], run_playbook_out) - - has_post = False - tasks = [] - early_publishers, late_publishers = self._transformPublishers( - self.jjb_job) - for publishers in [early_publishers, late_publishers]: - for publisher in publishers: - if 'scp' in publisher: - ret = self._makeSCPTask(publisher) - if ret['artifacts']: - has_artifacts = True - if ret['draft']: - has_draft = True - tasks.extend(ret['tasks']) - if 'afs' in builder: - self.log.error( - "Job {name} uses AFS publisher".format(name=self.name)) - if tasks: - has_post = True - play = collections.OrderedDict() - play['hosts'] = 'all' - play['tasks'] = tasks - with open(post_playbook, 'w') as post_playbook_out: - ordered_dump([play], post_playbook_out) - return has_artifacts, has_post, has_draft - - def toJobDict( - self, has_artifacts=False, has_post=False, has_draft=False, - project_names=[]): - output = collections.OrderedDict() - output['name'] = self.name - - expanded_projects = [] - if self.required_projects: - expanded_projects = expand_project_names( - self.required_projects, project_names) - # Look for project names in the job name. Lookie there - the - # python in operator works on lists and strings. - expanded_projects.extend(expand_project_names( - self.name, project_names)) - - output['parent'] = 'legacy-base' - if 'dsvm' in self.name: - output['parent'] = 'legacy-dsvm-base' - elif 'puppet-openstack-integration' in self.name: - output['parent'] = 'legacy-puppet-openstack-integration' - elif 'openstack/puppet-openstack-integration' in expanded_projects: - output['parent'] = 'legacy-puppet-openstack-integration' - elif has_artifacts: - output['parent'] = 'legacy-publish-openstack-artifacts' - elif has_draft: - output['success-url'] = 'html/' - output['run'] = os.path.join(self.job_path, 'run.yaml') - if has_post: - output['post-run'] = os.path.join(self.job_path, 'post.yaml') - - if self.vars: - output['vars'] = self.vars.copy() - - timeout = self.getTimeout() - if timeout: - output['timeout'] = timeout - - if self.nodes: - if len(self.nodes) == 1: - output['nodeset'] = self.getNodes()[0] - else: - output['nodeset'] = dict(nodes=self.getNodes()) - - if expanded_projects: - output['required-projects'] = sorted(list(set(expanded_projects))) - - if self.name in JOB_MATCHERS: - for k, v in JOB_MATCHERS[self.name].items(): - if k in output: - self.log.error( - 'Job %s has attributes directly and from matchers', - self.name) - output[k] = v - - return output - - def toPipelineDict(self): - if self.content: - output = self.content - else: - output = collections.OrderedDict() - output[self.name] = collections.OrderedDict() - - if self.parent: - output[self.name].setdefault('dependencies', self.getDepends()) - - if not self.voting: - output[self.name].setdefault('voting', False) - - if self.vars: - job_vars = output[self.name].get('vars', collections.OrderedDict()) - job_vars.update(self.vars) - - if self.branch: - output[self.name]['branches'] = self.branch - - if self.files: - output[self.name]['files'] = self.files - - if not output[self.name]: - return self.name - - return output - - -class JobMapping: - log = logging.getLogger("zuul.Migrate.JobMapping") - - def __init__(self, nodepool_config, layout, mapping_file=None): - self.layout = layout - self.job_direct = {} - self.labels = [] - self.job_mapping = [] - self.template_mapping = {} - self.jjb_jobs = {} - self.seen_new_jobs = [] - self.unshare = [] - nodepool_data = ordered_load(open(nodepool_config, 'r')) - for label in nodepool_data['labels']: - self.labels.append(label['name']) - if not mapping_file: - self.default_node = 'ubuntu-xenial' - else: - mapping_data = ordered_load(open(mapping_file, 'r')) - self.default_node = mapping_data['default-node'] - global SUFFIXES - SUFFIXES = mapping_data.get('strip-suffixes', []) - global SKIP_MACROS - SKIP_MACROS = mapping_data.get('skip-macros', []) - self.unshare = mapping_data.get('unshare', []) - for map_info in mapping_data.get('job-mapping', []): - if map_info['old'].startswith('^'): - map_info['pattern'] = re.compile(map_info['old']) - self.job_mapping.append(map_info) - else: - self.job_direct[map_info['old']] = map_info['new'] - - for map_info in mapping_data.get('template-mapping', []): - self.template_mapping[map_info['old']] = map_info['new'] - - def makeNewName(self, new_name, match_dict): - return new_name.format(**match_dict) - - def hasProjectTemplate(self, old_name): - return old_name in self.template_mapping - - def setJJBJobs(self, jjb_jobs): - self.jjb_jobs = jjb_jobs - - def getNewTemplateName(self, old_name): - return self.template_mapping.get(old_name, old_name) - - def mapNewJob(self, name, info) -> Optional[Job]: - matches = info['pattern'].search(name) - if not matches: - return None - match_dict = matches.groupdict() - if isinstance(info['new'], dict): - new_job = info['new'] - old_name = get_single_key(new_job) - new_name = old_name.format(**match_dict) - job = Job(orig=name, content={new_name: new_job[old_name]}) - else: - job = Job(orig=name, name=info['new'].format(**match_dict)) - - if 'vars' in info: - job.setVars(self._expandVars(info, match_dict)) - - return job - - def _expandVars(self, info, match_dict): - job_vars = info['vars'].copy() - for key in job_vars: - job_vars[key] = job_vars[key].format(**match_dict) - return job_vars - - def getNewJob(self, job_name, remove_gate): - if job_name in self.job_direct: - if isinstance(self.job_direct[job_name], dict): - job = Job(job_name, content=self.job_direct[job_name]) - else: - job = Job(job_name, name=self.job_direct[job_name]) - if job_name not in self.seen_new_jobs: - self.seen_new_jobs.append(self.job_direct[job_name]) - job.setNoEmit() - return job - - new_job = None - for map_info in self.job_mapping: - new_job = self.mapNewJob(job_name, map_info) - if new_job: - if job_name not in self.seen_new_jobs: - self.seen_new_jobs.append(new_job.name) - new_job.setNoEmit() - break - if not new_job: - orig_name = job_name - if remove_gate: - job_name = job_name.replace('gate-', '', 1) - job_name = 'legacy-{job_name}'.format(job_name=job_name) - new_job = Job(orig=orig_name, name=job_name) - - new_job.extractNode(self.default_node, self.labels) - - # Handle matchers - for layout_job in self.layout.get('jobs', []): - if re.search(layout_job['name'], new_job.orig): - # Matchers that can apply to templates must be processed first - # since project-specific matchers can cause the template to - # be expanded into a project. - if not layout_job.get('voting', True): - new_job.voting = False - if layout_job.get('branch'): - new_job.branch = layout_job['branch'] - if layout_job.get('files'): - new_job.files = layout_job['files'] - - new_job.addJJBJob(self.jjb_jobs) - return new_job - - -class ChangeQueue: - def __init__(self): - self.name = '' - self.assigned_name = None - self.generated_name = None - self.projects = [] - self._jobs = set() - - def getJobs(self): - return self._jobs - - def getProjects(self): - return [p.name for p in self.projects] - - def addProject(self, project): - if project not in self.projects: - self.projects.append(project) - self._jobs |= project.gate_jobs - - names = [x.name for x in self.projects] - names.sort() - self.generated_name = names[0].split('/')[-1] - - for job in self._jobs: - if job.queue_name: - if (self.assigned_name and - job.queue_name != self.assigned_name): - raise Exception("More than one name assigned to " - "change queue: %s != %s" % - (self.assigned_name, - job.queue_name)) - self.assigned_name = job.queue_name - self.name = self.assigned_name or self.generated_name - - def mergeChangeQueue(self, other): - for project in other.projects: - self.addProject(project) - - -class ZuulMigrate: - - log = logging.getLogger("zuul.Migrate") - - def __init__(self, layout, job_config, nodepool_config, - outdir, mapping, syntax_check): - self.layout = ordered_load(open(layout, 'r')) - self.job_config = job_config - self.outdir = outdir - self.mapping = JobMapping(nodepool_config, self.layout, mapping) - self.syntax_check = syntax_check - - self.jobs = {} - self.old_jobs = {} - self.job_objects = [] - self.new_templates = {} - - def run(self): - self.loadJobs() - self.buildChangeQueues() - self.convertJobs() - self.writeJobs() - - def loadJobs(self): - self.log.debug("Loading jobs") - builder = JJB() - builder.load_files([self.job_config]) - builder.parser.expandYaml() - unseen = set(self.jobs.keys()) - for job in builder.parser.jobs: - builder.expandMacros(job) - self.jobs[job['name']] = job - unseen.discard(job['name']) - for name in unseen: - del self.jobs[name] - self.mapping.setJJBJobs(self.jobs) - - def getOldJob(self, name): - if name not in self.old_jobs: - self.old_jobs[name] = OldJob(name) - return self.old_jobs[name] - - def flattenOldJobs(self, tree, name=None): - if isinstance(tree, str): - n = tree.format(name=name) - if n in self.mapping.unshare: - return [] - return [self.getOldJob(n)] - - new_list = [] # type: ignore - if isinstance(tree, list): - for job in tree: - new_list.extend(self.flattenOldJobs(job, name)) - elif isinstance(tree, dict): - parent_name = get_single_key(tree) - jobs = self.flattenOldJobs(tree[parent_name], name) - for job in jobs: - if job not in self.mapping.unshare: - new_list.append(self.getOldJob(job)) - if parent_name not in self.mapping.unshare: - new_list.append(self.getOldJob(parent_name)) - return new_list - - def buildChangeQueues(self): - self.log.debug("Building shared change queues") - - for j in self.layout['jobs']: - if '^' in j['name'] or '$' in j['name']: - continue - job = self.getOldJob(j['name']) - job.queue_name = j.get('queue-name') - - change_queues = [] - - for project in self.layout.get('projects'): - if 'gate' not in project: - continue - gate_jobs = set() - for template in project.get('template', []): - for pt in self.layout.get('project-templates'): - if pt['name'] != template['name']: - continue - if 'gate' not in pt['name']: - continue - gate_jobs |= set(self.flattenOldJobs(pt['gate'], - project['name'])) - gate_jobs |= set(self.flattenOldJobs(project['gate'])) - old_project = OldProject(project['name'], gate_jobs) - change_queue = ChangeQueue() - change_queue.addProject(old_project) - change_queues.append(change_queue) - self.log.debug("Created queue: %s" % change_queue) - - # Iterate over all queues trying to combine them, and keep doing - # so until they can not be combined further. - last_change_queues = change_queues - while True: - new_change_queues = self.combineChangeQueues(last_change_queues) - if len(last_change_queues) == len(new_change_queues): - break - last_change_queues = new_change_queues - - self.log.debug(" Shared change queues:") - for queue in new_change_queues: - self.log.debug(" %s containing %s" % ( - queue, queue.generated_name)) - self.change_queues = new_change_queues - - def combineChangeQueues(self, change_queues): - self.log.debug("Combining shared queues") - new_change_queues = [] - for a in change_queues: - merged_a = False - for b in new_change_queues: - if not a.getJobs().isdisjoint(b.getJobs()): - self.log.debug("Merging queue %s into %s" % (a, b)) - b.mergeChangeQueue(a) - merged_a = True - break # this breaks out of 'for b' and continues 'for a' - if not merged_a: - self.log.debug("Keeping queue %s" % (a)) - new_change_queues.append(a) - return new_change_queues - - def convertJobs(self): - pass - - def setupDir(self): - zuul_d = os.path.join(self.outdir, 'zuul.d') - job_outfile = os.path.join(zuul_d, 'zuul-legacy-jobs.yaml') - project_outfile = os.path.join(zuul_d, 'projects.yaml') - projects = [] - project_template_outfile = os.path.join( - zuul_d, 'zuul-legacy-project-templates.yaml') - if not os.path.exists(self.outdir): - os.makedirs(self.outdir) - if not os.path.exists(zuul_d): - os.makedirs(zuul_d) - elif os.path.exists(project_outfile): - projects = ordered_load(open(project_outfile, 'r')) - return dict( - job_outfile=job_outfile, - project_outfile=project_outfile, - projects=projects, - project_template_outfile=project_template_outfile, - ) - - def makeNewJobs(self, old_job, parent: Job=None): - self.log.debug("makeNewJobs(%s)", old_job) - if isinstance(old_job, str): - remove_gate = True - if old_job.startswith('gate-'): - # Check to see if gate- and bare versions exist - if old_job.replace('gate-', '', 1) in self.jobs: - remove_gate = False - job = self.mapping.getNewJob(old_job, remove_gate) - if parent: - job.setParent(parent) - return [job] - - new_list = [] # type: ignore - if isinstance(old_job, list): - for job in old_job: - new_list.extend(self.makeNewJobs(job, parent=parent)) - - elif isinstance(old_job, dict): - parent_name = get_single_key(old_job) - parent = self.makeNewJobs(parent_name, parent=parent)[0] - - jobs = self.makeNewJobs(old_job[parent_name], parent=parent) - for job in jobs: - new_list.append(job) - new_list.append(parent) - return new_list - - def writeProjectTemplate(self, template): - new_template = collections.OrderedDict() - if 'name' in template: - new_template['name'] = template['name'] - for key, value in template.items(): - if key == 'name': - continue - - # keep a cache of the Job objects so we can use it to get old - # job name to new job name when expanding templates into projects. - tmp = [job for job in self.makeNewJobs(value)] - self.job_objects.extend(tmp) - jobs = [job.toPipelineDict() for job in tmp] - new_template[key] = dict(jobs=jobs) - - return new_template - - def scanForProjectMatchers(self, project_name): - ''' Get list of job matchers that reference the given project name ''' - job_matchers = [] - for matcher in self.layout.get('jobs', []): - for skipper in matcher.get('skip-if', []): - if skipper.get('project'): - if re.search(skipper['project'], project_name): - job_matchers.append(matcher) - return job_matchers - - def findReferencedTemplateNames(self, job_matchers, project_name): - ''' Search templates in the layout file for matching jobs ''' - template_names = [] - - def search_jobs(template): - def _search(job): - if isinstance(job, str): - for matcher in job_matchers: - if re.search(matcher['name'], - job.format(name=project_name)): - template_names.append(template['name']) - return True - elif isinstance(job, list): - for i in job: - if _search(i): - return True - elif isinstance(job, dict): - for k, v in job.items(): - if _search(k) or _search(v): - return True - return False - - for key, value in template.items(): - if key == 'name': - continue - for job in template[key]: - if _search(job): - return - - for template in self.layout.get('project-templates', []): - search_jobs(template) - return template_names - - def expandTemplateIntoProject(self, template_name, project): - self.log.debug("EXPAND template %s into project %s", - template_name, project['name']) - # find the new template since that's the thing we're expanding - if template_name not in self.new_templates: - self.log.error( - "Template %s not found for expansion into project %s", - template_name, project['name']) - return - - template = self.new_templates[template_name] - - for pipeline, value in template.items(): - if pipeline == 'name': - continue - if pipeline not in project or 'jobs' not in project[pipeline]: - project[pipeline] = dict(jobs=[]) - project[pipeline]['jobs'].extend(value['jobs']) - - def getOldJobName(self, new_job_name): - for job in self.job_objects: - if job.name == new_job_name: - return job.orig - return None - - def applyProjectMatchers(self, matchers, project, final=False): - ''' - Apply per-project job matchers to the given project. - - :param matchers: Job matchers that referenced the given project. - :param project: The new project object. - ''' - - def processPipeline(pipeline_jobs, job_name_regex, files): - new_jobs = [] - for job in pipeline_jobs: - if isinstance(job, str): - old_job_name = self.getOldJobName(job) - if old_job_name and re.search( - job_name_regex, old_job_name): - self.log.debug( - "Applied irrelevant-files to job %s in project %s", - job, project['name']) - job = {job: {'irrelevant-files': - sorted(list(set(files)))}} - elif isinstance(job, dict): - job = job.copy() - job_name = get_single_key(job) - extras = job[job_name] - old_job_name = self.getOldJobName(job_name) - if old_job_name and re.search( - job_name_regex, old_job_name): - self.log.debug( - "Applied irrelevant-files to complex job " - "%s in project %s", job_name, project['name']) - if 'irrelevant-files' not in extras: - extras['irrelevant-files'] = [] - extras['irrelevant-files'].extend(files) - extras['irrelevant-files'] = sorted(list( - set(extras['irrelevant-files']))) - job[job_name] = extras - new_jobs.append(job) - return new_jobs - - def applyIrrelevantFiles(job_name_regex, files): - for k, v in project.items(): - if k in ('templates', 'name'): - continue - project[k]['jobs'] = processPipeline( - project[k].get('jobs', []), job_name_regex, files) - - if matchers: - for matcher in matchers: - # find the project-specific section - for skipper in matcher.get('skip-if', []): - if skipper.get('project'): - if re.search(skipper['project'], project['name']): - if 'all-files-match-any' in skipper: - applyIrrelevantFiles( - matcher['name'], - skipper['all-files-match-any']) - - if not final: - return - - for k, v in project.items(): - if k in ('templates', 'name'): - continue - jobs = [] - for job in project[k].get('jobs', []): - if isinstance(job, dict): - job_name = get_single_key(job) - else: - job_name = job - if job_name in JOB_MATCHERS: - jobs.append(job) - continue - orig_name = self.getOldJobName(job_name) - if not orig_name: - jobs.append(job) - continue - project_name = project['name'] - if '/' in project['name']: - project_name = project['name'].split('/')[1] - orig_name = orig_name.format(name=project_name) - info = {} - for layout_job in self.mapping.layout.get('jobs', []): - if 'parameter-function' in layout_job: - continue - if 'skip-if' in layout_job: - continue - if re.search(layout_job['name'], orig_name): - if not layout_job.get('voting', True): - info['voting'] = False - if layout_job.get('branch'): - info['branches'] = layout_job['branch'] - if layout_job.get('files'): - info['files'] = layout_job['files'] - if not isinstance(job, dict): - job = {job: info} - else: - job[job_name].update(info) - - jobs.append(job) - if jobs: - project[k]['jobs'] = jobs - - def writeProject(self, project): - ''' - Create a new v3 project definition. - - As part of creating the project, scan for project-specific job matchers - referencing this project and remove the templates matching the job - regex for that matcher. Expand the matched template(s) into the project - so we can apply the project-specific matcher to the job(s). - ''' - new_project = collections.OrderedDict() - if 'name' in project: - new_project['name'] = project['name'] - - exp_template_names = TEMPLATES_TO_EXPAND.get(project['name'], []) - - templates_to_expand = [] - if 'template' in project: - new_project['templates'] = [] - for template in project['template']: - if template['name'] in exp_template_names: - templates_to_expand.append(template['name']) - continue - new_project['templates'].append( - self.mapping.getNewTemplateName(template['name'])) - - for key, value in project.items(): - if key in ('name', 'template'): - continue - else: - new_project[key] = collections.OrderedDict() - if key == 'gate': - for queue in self.change_queues: - if (project['name'] not in queue.getProjects() or - len(queue.getProjects()) == 1): - continue - new_project[key]['queue'] = queue.name - tmp = [job for job in self.makeNewJobs(value)] - # Don't insert into self.job_objects - that was done - # in the speculative pass - jobs = [job.toPipelineDict() for job in tmp] - if jobs: - new_project[key]['jobs'] = jobs - if not new_project[key]: - del new_project[key] - - for name in templates_to_expand: - self.expandTemplateIntoProject(name, new_project) - - job_matchers = self.scanForProjectMatchers(project['name']) - - # Need a deep copy after expansion, else our templates end up - # also getting this change. - new_project = copy.deepcopy(new_project) - self.applyProjectMatchers(job_matchers, new_project, final=True) - - return new_project - - def checkSpeculativeProject(self, project): - ''' - Create a new v3 project definition expanding all templates. - ''' - new_project = collections.OrderedDict() - if 'name' in project: - new_project['name'] = project['name'] - - templates_to_expand = [] - for template in project.get('template', []): - templates_to_expand.append(template['name']) - - # We have to do this section to expand self.job_objects - for key, value in project.items(): - if key in ('name', 'template'): - continue - else: - new_project[key] = collections.OrderedDict() - if key == 'gate': - for queue in self.change_queues: - if project['name'] not in queue.getProjects(): - continue - if len(queue.getProjects()) == 1: - continue - new_project[key]['queue'] = queue.name - tmp = [job for job in self.makeNewJobs(value)] - self.job_objects.extend(tmp) - - for name in templates_to_expand: - - expand_project = copy.deepcopy(new_project) - self.expandTemplateIntoProject(name, expand_project) - - # Need a deep copy after expansion, else our templates end up - # also getting this change. - expand_project = copy.deepcopy(expand_project) - job_matchers = self.scanForProjectMatchers(project['name']) - self.applyProjectMatchers(job_matchers, expand_project) - - # We should now have a project-pipeline with only the - # jobs expanded from this one template - for project_part in expand_project.values(): - # The pipelines are dicts - we only want pipelines - if isinstance(project_part, dict): - if 'jobs' not in project_part: - continue - self.processProjectTemplateExpansion( - project_part, project, name) - - def processProjectTemplateExpansion(self, project_part, project, template): - # project_part should be {'jobs': []} - job_list = project_part['jobs'] - for new_job in job_list: - if isinstance(new_job, dict): - new_job_name = get_single_key(new_job) - info = new_job[new_job_name] - else: - new_job_name = new_job - info = None - orig_name = self.getOldJobName(new_job_name) - if not orig_name: - self.log.error("Job without old name: %s", new_job_name) - continue - project_name = project['name'] - if '/' in project['name']: - project_name = project['name'].split('/')[1] - orig_name = orig_name.format(name=project_name) - - for layout_job in self.mapping.layout.get('jobs', []): - if 'parameter-function' in layout_job: - continue - if re.search(layout_job['name'], orig_name): - if not info: - info = {} - if not layout_job.get('voting', True): - info['voting'] = False - if layout_job.get('branch'): - info['branches'] = layout_job['branch'] - if layout_job.get('files'): - info['files'] = layout_job['files'] - - if info: - expansion = dict(info=info, template=template) - JOBS_FOR_EXPAND[new_job_name][project['name']] = expansion - - def writeJobs(self): - output_dir = self.setupDir() - job_outfile = output_dir['job_outfile'] - project_outfile = output_dir['project_outfile'] - project_config = output_dir['projects'] - project_dicts = list_to_project_dicts(project_config) - project_template_outfile = output_dir['project_template_outfile'] - job_config = [] - template_config = [] - - for template in self.layout.get('project-templates', []): - self.log.debug("Processing template: %s", template) - new_template = self.writeProjectTemplate(template) - self.new_templates[new_template['name']] = new_template - if not self.mapping.hasProjectTemplate(template['name']): - template_config.append({'project-template': new_template}) - template_config = sorted( - template_config, - key=lambda template: template['project-template']['name']) - - for project in self.layout.get('projects', []): - self.checkSpeculativeProject(project) - normalize_project_expansions() - - project_names = [] - for project in self.layout.get('projects', []): - project_names.append(project['name']) - project_dict = self.writeProject(project) - merge_project_dict( - project_dicts, project['name'], - project_dict) - project_config = project_dicts_to_list(project_dicts) - - seen_jobs = [] - for job in sorted(self.job_objects, key=lambda job: job.name): - if (job.name not in seen_jobs and - job.name not in self.mapping.seen_new_jobs and - job.emit): - has_artifacts, has_post, has_draft = job.emitPlaybooks( - self.outdir, self.syntax_check) - job_config.append({'job': job.toJobDict( - has_artifacts, has_post, has_draft, project_names)}) - seen_jobs.append(job.name) - job_config = sorted( - job_config, key=lambda job: job['job']['name']) - - with open(job_outfile, 'w') as yamlout: - # Insert an extra space between top-level list items - yamlout.write(ordered_dump(job_config).replace('\n-', '\n\n-')) - - with open(project_template_outfile, 'w') as yamlout: - # Insert an extra space between top-level list items - yamlout.write( - ordered_dump(template_config).replace('\n-', '\n\n-')) - - with open(project_outfile, 'w') as yamlout: - # Insert an extra space between top-level list items - yamlout.write(ordered_dump(project_config).replace('\n-', '\n\n-')) - - -def main(): - yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - construct_yaml_map) - - yaml.add_representer(collections.OrderedDict, project_representer, - Dumper=IndentedDumper) - yaml.representer.BaseRepresenter.represent_scalar = my_represent_scalar - - parser = argparse.ArgumentParser(description=DESCRIPTION) - parser.add_argument( - 'layout', - help="The Zuul v2 layout.yaml file to read.") - parser.add_argument( - 'job_config', - help="Directory containing Jenkins Job Builder job definitions.") - parser.add_argument( - 'nodepool_config', - help="Nodepool config file containing complete set of node names") - parser.add_argument( - 'outdir', - help="A directory into which the Zuul v3 config will be written.") - parser.add_argument( - '--mapping', - default=None, - help="A filename with a yaml mapping of old name to new name.") - parser.add_argument( - '-v', dest='verbose', action='store_true', help='verbose output') - parser.add_argument( - '--syntax-check', dest='syntax_check', action='store_true', - help='Run ansible-playbook --syntax-check on generated playbooks') - args = parser.parse_args() - if args.verbose: - logging.basicConfig(level=logging.DEBUG) - else: - logging.basicConfig(level=logging.INFO) - - ZuulMigrate(args.layout, args.job_config, args.nodepool_config, - args.outdir, args.mapping, args.syntax_check).run()