Implement Gerrit change retrieval and processing

README.md

@@ -1,2 +1,2 @@
 # Gerrit-to-Github-Issues
-Set of scripts that can be used to synchronize/update Github Issues with data from a Gerrit instance
+Set of scripts that can be used to synchronize/update Github Issues with data from a Gerrit instance.

gerrit_to_github_issues/__main__.py

@@ -1,58 +1,64 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Licensed under the Apache License, Version 2.0 (the 'License');
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
+# distributed under the License is distributed on an 'AS IS' BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import argparse
 import logging
-import re
+import os
-import sys
 
-import github
+import errors
+from engine import update
+
+LOG_FORMAT = '%(asctime)s %(levelname)-8s %(name)s:' \
+             '%(funcName)s [%(lineno)3d] %(message)s'  # noqa
+
+LOG = logging.getLogger(__name__)
+
+
+def validate(namespace: argparse.Namespace):
+    arg_dict = vars(namespace)
+    if not ((arg_dict['github_username'] and arg_dict['github_password']) or arg_dict['github_token']):
+        raise errors.GithubConfigurationError
+    return arg_dict
 
-GH_USER = sys.argv[1]
-GH_PW = sys.argv[2]
-ZUUL_MESSAGE = sys.argv[3]
-GERRIT_URL = sys.argv[4]
-REPO_NAME = 'airshipit/airshipctl'
 
 if __name__ == '__main__':
-    issue_number = re.search(r'(?<=\[#)(.*?)(?=\])', ZUUL_MESSAGE).group(0)
+    parser = argparse.ArgumentParser(
-    gh = github.Github(GH_USER, GH_PW)
+        prog='gerrit-to-github-issues',
-    repo = gh.get_repo(REPO_NAME)
+        usage='synchronizes GitHub Issues with new changes found in Gerrit',
-    issue = repo.get_issue(number=int(issue_number))
+        description='This script evaluates the following logic on open changes from Gerrit:\n'
-    comment_msg = ''
+                    '1. Check for and extract an issue tag (i.e. "[#3]") from the open change\'s commit message.\n'
-    link_exists = False
+                    '2. Check associated Github Issue for a link to the change. If no such link exists, comment it.\n'
-    for comment in issue.get_comments():
+                    '3. If the associated issue was closed, re-open it and comment on it describing why it was '
-        if GERRIT_URL in comment.body:
+                    're-opened and a link to the Gerrit change that was found.\n'
-            logging.log(logging.INFO, 'Gerrit link has already been posted')
+                    '4. If the Gerrit change\'s commit message contains a "WIP" or "DNM" tag, add the "wip" label and '
-            link_exists = True
+                    'to the issue remove other process labels such as "ready for review".\n'
-    if issue.state == 'closed' and not link_exists:
+                    '5. If no "WIP" or "DNM" tag is found in the change\'s commit message, add the "ready for review" '
-        issue.edit(state='open')
+                    'label to the issue and remove other process labels such as "ready for review".'
-        comment_msg += 'Issue reopened due to new activity on Gerrit.\n\n'
+    )
-    if 'WIP' in ZUUL_MESSAGE.upper() or 'DNM' in ZUUL_MESSAGE.upper():
+    parser.add_argument('-g', '--gerrit-url', action='store', required=True, type=str,
-        logging.log(logging.INFO, 'Changing status with `wip` label')
+                        default=os.getenv('GERRIT_URL', default=None), help='Target Gerrit URL.')
-        issue.add_to_labels('wip')
+    parser.add_argument('-u', '--github-username', action='store', required=False, type=str,
-        try:
+                        default=os.getenv('GITHUB_USER', default=None),
-            issue.remove_from_labels('ready for review')
+                        help='Username to use for GitHub Issues integration. Defaults to GITHUB_USER in '
-        except github.GithubException:
+                             'environmental variables. Must be used with a password.')
-            logging.log(logging.DEBUG, 'Could not remove `ready for review` label, '
+    parser.add_argument('-p', '--github-password', action='store', required=False, type=str,
-                                       'it probably was not on the issue')
+                        default=os.getenv('GITHUB_PW', default=None),
-    else:
+                        help='Password to use for GitHub Issues integration. Defaults to GITHUB_PW in '
-        logging.log(logging.INFO, 'Changing status with `ready for review` label')
+                             'environmental variables. Must be used with a username.')
-        issue.add_to_labels('ready for review')
+    parser.add_argument('-t', '--github-token', action='store', required=False, type=str,
-        try:
+                        default=os.getenv('GITHUB_TOKEN', default=None),
-            issue.remove_from_labels('wip')
+                        help='Token to use for GitHub Issues integration. Defaults to GITHUB_TOKEN in '
-        except github.GithubException:
+                             'environmental variables. This will be preferred over a username/password.')
-            logging.log(logging.DEBUG, 'Could not remove `wip` label, '
+    parser.add_argument('gerrit_project_name', action='store', type=str, help='Target Gerrit project.')
-                                       'it probably was not on the issue')
+    parser.add_argument('github_project_name', action='store', type=str, help='Target Github project.')
-    if not link_exists:
+    ns = parser.parse_args()
-        comment_msg += f'New Related Change: {GERRIT_URL}'
+    args = validate(ns)
-    if comment_msg:
+    update(**args)
-        issue.create_comment(comment_msg)
-        logging.log(logging.INFO, f'Comment posted to issue #{issue_number}')

gerrit_to_github_issues/engine.py

@@ -0,0 +1,68 @@
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+
+import github
+from github.Repository import Repository
+
+import gerrit
+import github_issues
+
+LOG = logging.getLogger(__name__)
+
+
+def update(gerrit_url: str, gerrit_project_name: str, github_project_name: str, github_user: str, github_pw: str,
+           github_token: str):
+    repo = github_issues.get_repo(github_project_name, github_user, github_pw, github_token)
+    change_list = gerrit.get_changes(gerrit_url, gerrit_project_name)
+    for change in change_list:
+        if 'commitMessage' in change:
+            process_change(change, repo, gerrit_url)
+
+
+def process_change(change: dict, repo: Repository, gerrit_url: str):
+    issue_number = github_issues.parse_issue_number(change['commitMessage'])
+    if not issue_number:
+        LOG.warning(f'No issue tag found for change #{change["number"]}')
+        return
+    try:
+        issue = repo.get_issue(issue_number)
+    except github.GithubException:
+        LOG.warning(f'Issue #{issue_number} not found for project')
+        return
+    comment_msg = ''
+    change_url = gerrit.make_gerrit_url(gerrit_url, change['number'])
+    link_exists = github_issues.check_issue_for_matching_comments(issue, change_url)
+    if issue.state == 'closed' and not link_exists:
+        issue.edit(state='open')
+        comment_msg += 'Issue reopened due to new activity on Gerrit.\n\n'
+    if 'WIP' in change['commitMessage'] or 'DNM' in change['commitMessage']:
+        LOG.debug(f'add `wip` to {issue_number}')
+        #issue.add_to_labels('wip')
+        try:
+            LOG.debug(f'rm `ready for review` to {issue_number}')
+            #issue.remove_from_labels('ready for review')
+        except github.GithubException:
+            LOG.debug(f'`ready for review` tag does not exist on issue #{issue_number}')
+    else:
+        LOG.debug(f'add `ready for review` to {issue_number}')
+        #issue.add_to_labels('ready for review')
+        try:
+            LOG.debug(f'rm `wip` to {issue_number}')
+            #issue.remove_from_labels('wip')
+        except github.GithubException:
+            LOG.debug(f'`wip` tag does not exist on issue #{issue_number}')
+    if not link_exists:
+        comment_msg += f'New Related Change: {gerrit_url}'
+    if comment_msg:
+        #issue.create_comment(comment_msg)
+        LOG.info(f'Comment posted to issue #{gerrit_url}')

gerrit_to_github_issues/errors.py

@@ -0,0 +1,19 @@
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class GithubConfigurationError(Exception):
+    message = 'No Github username/password or token has been defined or they are invalid.'
+
+
+class GerritConfigurationError(Exception):
+    message = 'No Gerrit URL defined.'

gerrit_to_github_issues/gerrit.py

@@ -0,0 +1,26 @@
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+
+from fabric import Connection
+
+
+def get_changes(gerrit_url: str, project_name: str, port: int = 29418) -> list:
+    cmd = f'gerrit query --format=JSON status:open project:{project_name}'
+    result = Connection(gerrit_url, port=port).run(cmd)
+    processed_stdout = '{"data":[%s]}' % ','.join(list(filter(None, result.stdout.split('\n'))))
+    data = json.loads(processed_stdout)
+    return data
+
+
+def make_gerrit_url(gerrit_url: str, change_number: str, protocol: str = 'https'):
+    return f'{protocol}://{gerrit_url}/{change_number}'

gerrit_to_github_issues/github_issues.py

@@ -0,0 +1,42 @@
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import re
+
+import github
+from github.Issue import Issue
+from github.Repository import Repository
+
+import errors
+
+
+def parse_issue_number(commit_msg: str) -> int:
+    match = re.search(r'(?<=\[#)(.*?)(?=\])', commit_msg)
+    if not match:
+        return None
+    return int(match.group(0))
+
+
+def get_repo(repo_name: str, github_user: str, github_pw: str, github_token: str) -> Repository:
+    if github_token:
+        gh = github.Github(github_token)
+    elif github_user and github_pw:
+        gh = github.Github(github_user, github_pw)
+    else:
+        raise errors.GithubConfigurationError
+    return gh.get_repo(repo_name)
+
+
+def check_issue_for_matching_comments(issue: Issue, contains: str) -> bool:
+    for comment in issue.get_comments():
+        if contains in comment.body:
+            return True
+    return False

requirements.txt

@@ -1 +1,2 @@
 PyGithub==1.46
+fabric==2.5.0