diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst index 9f2236fb9f5..cefb188ff8a 100644 --- a/doc/source/contributor/index.rst +++ b/doc/source/contributor/index.rst @@ -77,6 +77,14 @@ Neutron Internals internals/index modules +OVN Driver +---------- + +.. toctree:: + :maxdepth: 2 + + ovn/index + Dashboards ---------- diff --git a/doc/source/contributor/ovn/index.rst b/doc/source/contributor/ovn/index.rst new file mode 100644 index 00000000000..06716820148 --- /dev/null +++ b/doc/source/contributor/ovn/index.rst @@ -0,0 +1,11 @@ +.. meta:: + :keywords: ovn, networking-ovn, OpenStack, neutron + +=========== +OVN backend +=========== + +.. toctree:: + :maxdepth: 1 + + tools.rst diff --git a/doc/source/contributor/ovn/tools.rst b/doc/source/contributor/ovn/tools.rst new file mode 100644 index 00000000000..f342f3f8c5a --- /dev/null +++ b/doc/source/contributor/ovn/tools.rst @@ -0,0 +1,109 @@ +.. _ovn_tools: + +OVN Tools +========= + +This document offers details on Neutron tools available for assisting +with using the Open Virtual Network (OVN) backend. + +Patches and Cherry-picks +------------------------ + +Overview +^^^^^^^^ +As described in the +`ovn-migration blueprint `__, +Neutron's OVN ML2 plugin has merged to the Neutron repository as of the Ussuri +release. With that, special care must be taken to apply Neutron +changes to the proper stable branches of the networking-ovn repo. + +.. note:: + + These scripts are generic enough to work on any patch file, but + particularly handy with the networking-ovn migration. + + +tools/files_in_patch.py +^^^^^^^^^^^^^^^^^^^^^^^ +Use this to show files that are changed in a patch file. + +.. code-block:: console + + $ # Make a patch to use as example + $ git show > /tmp/commit.patch + + $ ./tools/files_in_patch.py /tmp/commit.patch | grep .py + tools/download_gerrit_change.py + tools/files_in_patch.py + tools/migrate_names.py + + +tools/download_gerrit_change.py +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This tool is needed by ``migrate_names.py`` (see below), but it can be used +independently. Given a Gerrit change id, it will fetch the latest +patchset of the change from `review.opendev.org `__ +as a patch file. The output can be stdout or an optional filename. + +.. code-block:: console + + $ ./tools/download_gerrit_change.py --help + Usage: download_gerrit_change.py [OPTIONS] GERRIT_CHANGE + + Options: + -o, --output_patch TEXT Output patch file. Default: stdout + -g, --gerrit_url TEXT The url to Gerrit server [default: + https://review.opendev.org/] + -t, --timeout INTEGER Timeout, in seconds [default: 10] + --help Show this message and exit. + + $ ./tools/download_gerrit_change.py 698863 -o /tmp/change.patch + $ ./tools/files_in_patch.py /tmp/change.patch + networking_ovn/ml2/mech_driver.py + networking_ovn/ml2/trunk_driver.py + networking_ovn/tests/unit/ml2/test_mech_driver.py + networking_ovn/tests/unit/ml2/test_trunk_driver.py + + +tools/migrate_names.py +^^^^^^^^^^^^^^^^^^^^^^ + +Use this tool to modify the name of the files in a patchfile so it can +be converted to/from the +`legacy networking-ovn `__ and +`Neutron `__ repositories. + +The mapping of how the files are renamed is based on ``migrate_names.txt``, which is located +in the same directory where ``migrate_names.py`` is installed. That behavior can be modified +via the ``--mapfile`` option. More information on how the map is parsed is provided in the header +section of that file. + +.. code-block:: console + + $ ./tools/migrate_names.py --help + Usage: migrate_names.py [OPTIONS] + + Options: + -i, --input_patch TEXT input_patch patch file or gerrit change + -o, --output_patch TEXT Output patch file. Default: stdout + -m, --mapfile PATH Data file that specifies mapping to be applied to + input [default: /home/user/openstack/neutron.git + /tools/migrate_names.txt] + --reverse / --no-reverse Map filenames from networking-ovn to Neutron repo + --help Show this message and exit. + $ ./tools/migrate_names.py -i 701646 > /tmp/ovn_change.patch + $ ./tools/migrate_names.py -o /tmp/reverse.patch -i /tmp/ovn_change.patch --reverse + $ diff /tmp/reverse.patch /tmp/ovn_change.patch | grep .py + < --- a/neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py + < +++ b/neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py + > --- a/networking_ovn/ml2/mech_driver.py + > +++ b/networking_ovn/ml2/mech_driver.py + <... snip ...> + + $ ./tools/files_in_patch.py /tmp/ovn_change.patch + networking_ovn/ml2/mech_driver.py + networking_ovn/ml2/trunk_driver.py + networking_ovn/tests/unit/ml2/test_mech_driver.py + networking_ovn/tests/unit/ml2/test_trunk_driver.py + diff --git a/tools/download_gerrit_change.py b/tools/download_gerrit_change.py new file mode 100755 index 00000000000..f21f93370c6 --- /dev/null +++ b/tools/download_gerrit_change.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import base64 + +import click +import requests + +GERRIT_URL = 'https://review.opendev.org/' +TIMEOUT = 10 + + +def fetch(change, output_patch=None, url=GERRIT_URL, timeout=TIMEOUT): + params = {'download': None} + r = requests.get( + url='{}/changes/{}/revisions/current/patch'.format(url, change), + params=params, + timeout=timeout) + r.raise_for_status() + message_bytes = base64.b64decode(r.text) + if output_patch and output_patch != '-': + with open(output_patch, 'wb') as output_fd: + output_fd.write(message_bytes) + return str(message_bytes, 'utf-8') + + +@click.command() +@click.argument('gerrit_change', nargs=1, type=click.INT) +@click.option('-o', '--output_patch', + help='Output patch file [default: stdout]') +@click.option('-g', '--gerrit_url', + default=GERRIT_URL, + show_default=True, + help='The url to Gerrit server') +@click.option('-t', '--timeout', + default=TIMEOUT, + show_default=True, + type=click.INT, + help='Timeout, in seconds') +def cli(gerrit_change, output_patch, gerrit_url, timeout): + message = fetch(gerrit_change, output_patch, gerrit_url, timeout) + if not output_patch or output_patch == '-': + print(message) + + +if __name__ == '__main__': + cli() diff --git a/tools/files_in_patch.py b/tools/files_in_patch.py new file mode 100755 index 00000000000..30bfb1c6f0d --- /dev/null +++ b/tools/files_in_patch.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import re +import sys + +file_names = set() + + +def parse_input(input_file): + global file_names + + while True: + line_buffer = input_file.readline() + if not line_buffer: + break + line_match = re.search(r"^\s*---\s+([^\s@]+)[\s@]+", line_buffer) + if not line_match: + line_match = re.search(r"^\s*\+\+\+\s+([^\s@]+)[\s@]+", + line_buffer) + if line_match: + curr_file_name = line_match.group(1) + + # trim off 'a/' and 'b/' that you will normally see in git output + # + if len(curr_file_name) > 2 and curr_file_name[1] == '/' and ( + curr_file_name[0] == 'a' or curr_file_name[0] == 'b'): + curr_file_name = curr_file_name[2:] + + file_names.add(curr_file_name) + + +def prune_unwanted_names(): + global file_names + + unwanted_names = set(['/dev/null']) + + for curr_file_name in file_names: + # ignore files that end in '.orig' as long as non-.orig exists + line_match = re.search(r"^(.+)\.[oO][Rr][iI][gG]$", curr_file_name) + if line_match and line_match.group(1) in file_names: + unwanted_names.add(curr_file_name) + continue + + file_names -= unwanted_names + + +def print_file_names(): + for name in sorted(file_names): + print(name) + + +if __name__ == '__main__': + if len(sys.argv) == 1: + parse_input(sys.stdin) + else: + for curr_input_name in sys.argv[1:]: + try: + with open(curr_input_name, 'r') as curr_input_file: + parse_input(curr_input_file) + except IOError as e_str: + sys.stderr.write( + "Cannot open {}: {}\n".format(curr_input_name, e_str)) + sys.exit(255) + + prune_unwanted_names() + print_file_names() diff --git a/tools/migrate_names.py b/tools/migrate_names.py new file mode 100755 index 00000000000..818cdec4cb4 --- /dev/null +++ b/tools/migrate_names.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from collections import namedtuple +import contextlib +import os +import re +import sys + +import click + +import download_gerrit_change + +root_dir = os.path.dirname(os.path.realpath(__file__)) +Migration = namedtuple('Migration', 'from_repo to_repo') + + +def read_mapfile(mapfile): + dirmaps = [] + with open(mapfile, 'r') as mapfile_fd: + for line_buffer in mapfile_fd.readlines(): + # ignore empty lines and anything after # + line_match = re.search("^([^#]+)", line_buffer.strip()) + if not line_match: + continue + line_buffer = line_match.group(1) + # look for tuple of 2 elements + line_match = re.search(r"^([^\s]+)\s+(.+)", line_buffer.strip()) + if not line_match: + continue + ovn_match, neutron_match = line_match.group(1), line_match.group(2) + dirmaps.append(Migration(neutron_match, ovn_match)) + return dirmaps + + +def parse_input(dirmaps, patch_content, output_fd): + for line_buffer in patch_content.splitlines(): + # locate markers in patch file for filenames and see if they need + # to me renamed based on dirmaps + filename_replaced = False + line_match = re.search(r"^\s*---\s+([^\s@]+)[\s@]*", line_buffer) + if not line_match: + line_match = re.search(r"^\s*\+\+\+\s+([^\s@]+)[\s@]*", + line_buffer) + if line_match: + for old, new in dirmaps: + new_line_buffer = line_buffer.replace(old, new) + if new_line_buffer != line_buffer: + filename_replaced = True + output_fd.write("{}\n".format(new_line_buffer)) + break + if not filename_replaced: + output_fd.write("{}\n".format(line_buffer)) + + +@contextlib.contextmanager +def open_output(filename=None): + if filename and filename != '-': + fh = open(filename, 'w') + else: + fh = sys.stdout + try: + yield fh + finally: + if fh is not sys.stdout: + fh.close() + + +@click.command() +@click.option('-i', '--input_patch', prompt='Input patch file or gerrit id', + help='input_patch patch file or gerrit change') +@click.option('-o', '--output_patch', default='-', + help='Output patch file. Default: stdout') +@click.option('-m', '--mapfile', + default=os.path.join(root_dir, 'migrate_names.txt'), + show_default=True, + type=click.Path(), + help='Data file that specifies mapping to be applied to input') +@click.option('--reverse/--no-reverse', + default=False, + help='Map filenames from networking-ovn to Neutron repo') +def cli(input_patch, output_patch, mapfile, reverse): + dirmaps = read_mapfile(mapfile) + if reverse: + dirmaps = [Migration(two, one) for one, two in dirmaps] + if os.path.isfile(input_patch): + with open(input_patch, 'r') as input_fd: + patch_content = ''.join(input_fd.readlines()) + else: + patch_content = download_gerrit_change.fetch(input_patch) + + with open_output(output_patch) as output_fd: + parse_input(dirmaps, patch_content, output_fd) + + +if __name__ == '__main__': + cli() diff --git a/tools/migrate_names.txt b/tools/migrate_names.txt new file mode 100644 index 00000000000..937fb7d542e --- /dev/null +++ b/tools/migrate_names.txt @@ -0,0 +1,46 @@ +# This file provides a list of tuples that represent how the files +# in Networking-OVN repo are mapped to/from Neutron repo, as part of the blue +# print documented in: +# +# https://review.opendev.org/#/c/658414/ (specs/ussuri/ml2ovs-ovn-convergence.rst) +# +# Also see: +# https://ethercalc.openstack.org/networking-ovn-migration +# https://review.opendev.org/#/q/topic:bp/neutron-ovn-merge+-is:abandoned +# +# Empty lines and anything after # are ignored. +# The 2 columns in this tile are added as a tuple in a list of +# files and directories to be mapped. More specific lines must be listed +# above less specific lines as the mapping stops on the first match. +# +# Networking-OVN Neutron + +devstack/lib/ovn devstack/lib/ovn_agent +networking_ovn/ovsdb/impl_idl_ovn.py neutron/ovsdb/impl_idl_ovn.py +networking_ovn/ovsdb neutron/ovsdb/ovn +networking_ovn/ovn_db_sync.py neutron/plugins/ml2/drivers/ovn/mech_driver/ovsdb/ovn_db_sync.py +networking_ovn/agent/metadata neutron/agent/ovn/metadata +networking_ovn/cmd neutron/cmd +networking_ovn/common/config.py neutron/conf/ovn.py +networking_ovn/common/acl.py neutron/plugins/ml2/drivers/ovn/common/acl.py +networking_ovn/common/constants.py neutron/common/ovn/constants.py +networking_ovn/common/exceptions.py neutron/common/ovn/exceptions.py +networking_ovn/common/hash_ring_manager.py neutron/common/ovn/hash_ring_manager.py +networking_ovn/common/maintanance.py neutron/plugins/ml2/drivers/ovn/mech_driver/ovsdb/maintenance.py +networking_ovn/common/ovn_client.py neutron/plugins/ml2/drivers/ovn/common/client.py +networking_ovn/common/utils.py neutron/common/ovn/utils.py +networking_ovn/conf/agent/metadata neutron/conf/agent/ovn/metadata.py +networking_ovn/db neutron/db +networking_ovn/ml2/mech_driver.py neutron/plugins/ml2/drivers/ovn/mech_driver/mech_driver.py +networking_ovn/ml2/qos_driver.py neutron/services/qos/drivers/ovn/driver.py +networking_ovn/ml2/trunk_driver.py neutron/services/trunk/drivers/ovn/trunk_driver.py +networking_ovn/l3/l3_ovn.py neutron/services/ovn_l3/l3_ovn.py +networking_ovn/l3/l3_ovn_scheduler.py neutron/scheduler/ovn_l3_scheduler.py +networking_ovn/tests/unit/ml2/test_mech_driver.py neutron/tests/unit/plugins/ml2/drivers/ovn/mech_driver/test_mech_driver.py +networking_ovn/tests/unit/ml2/test_qos_driver.py neutron/tests/unit/services/qos/drivers/ovn/test_driver.py +networking_ovn/tests/unit/ml2/test_trunk_driver.py neutron/tests/unit/services/trunk/drivers/ovn/test_trunk_driver.py +networking_ovn/tests neutron/tests +networking_ovn/common/extensions.py neutron/extensions/ovn.py +networking_ovn/tests/unit/fakes.py neutron/tests/unit/fake_resources.py +migration tools/migration_to_ovn + diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 00000000000..e43ec246e23 --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1,5 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +click>=7.0 # BSD +requests>=2.14.2 # Apache-2.0