From 344b8da469b8754fb23949315313e646a64dbb74 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 27 Jul 2011 11:21:34 -0400 Subject: [PATCH] Added script to work export github issues. Change-Id: Ibfabddf576d3317fa68c736f1f8ebc7cd78768a1 --- bug-export.rnc | 97 +++++++++++++++++++++++++++++++++++ gh_issues_to_lp_bugs.py | 111 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+) create mode 100644 bug-export.rnc create mode 100644 gh_issues_to_lp_bugs.py diff --git a/bug-export.rnc b/bug-export.rnc new file mode 100644 index 00000000..c4635a75 --- /dev/null +++ b/bug-export.rnc @@ -0,0 +1,97 @@ +default namespace = "https://launchpad.net/xmlns/2006/bugs" + +start = lpbugs + +# Data types + +boolean = "True" | "False" +lpname = xsd:string { pattern = "[a-z0-9][a-z0-9\+\.\-]*" } +cvename = xsd:string { pattern = "(19|20)[0-9][0-9]-[0-9][0-9][0-9][0-9]" } + +# XXX: jamesh 2006-04-11 bug=105401: +# These status and importance values need to be kept in sync with the +# rest of Launchpad. However, there are not yet any tests for this. +# https://bugs.launchpad.net/bugs/105401 +status = ( + "NEW" | + "INCOMPLETE" | + "INVALID" | + "WONTFIX" | + "CONFIRMED" | + "TRIAGED" | + "INPROGRESS" | + "FIXCOMMITTED" | + "FIXRELEASED" | + "UNKNOWN") +importance = ( + "UNKNOWN" | + "CRITICAL" | + "HIGH" | + "MEDIUM" | + "LOW" | + "WISHLIST" | + "UNDECIDED") + +# Content model for a person element. The element content is the +# person's name. For successful bug import, an email address must be +# provided. +person = ( + attribute name { lpname }?, + attribute email { text }?, + text) + +lpbugs = element launchpad-bugs { bug* } + +bug = element bug { + attribute id { xsd:integer } & + element private { boolean }? & + element security_related { boolean }? & + element duplicateof { xsd:integer }? & + element datecreated { xsd:dateTime } & + element nickname { lpname }? & + # The following will likely be renamed summary in a future version. + element title { text } & + element description { text } & + element reporter { person } & + element status { status } & + element importance { importance } & + element milestone { lpname }? & + element assignee { person }? & + element urls { + element url { attribute href { xsd:anyURI }, text }* + }? & + element cves { + element cve { cvename }* + }? & + element tags { + element tag { lpname }* + }? & + element bugwatches { + element bugwatch { attribute href { xsd:anyURI } }* + }? & + element subscriptions { + element subscriber { person }* + }? & + comment+ +} + +# A bug has one or more comments. The first comment duplicates the +# reporter, datecreated, title, description of the bug. +comment = element comment { + element sender { person } & + element date { xsd:dateTime } & + element title { text }? & + element text { text } & + attachment* +} + +# A bug attachment. Attachments are associated with a bug comment. +attachment = element attachment { + attribute href { xsd:anyURI }? & + element type { "PATCH" | "UNSPECIFIED" }? & + element filename { text }? & + # The following will likely be renamed summary in a future version. + element title { text }? & + element mimetype { text }? & + element contents { xsd:base64Binary } +} diff --git a/gh_issues_to_lp_bugs.py b/gh_issues_to_lp_bugs.py new file mode 100644 index 00000000..adecacbf --- /dev/null +++ b/gh_issues_to_lp_bugs.py @@ -0,0 +1,111 @@ +from xml.sax.saxutils import escape +from contextlib import closing +import codecs +import simplejson +import urllib2 +import os +import sys +import time + +if len(sys.argv) != 3: + print "A team/user and a project/repo are required arguments" + sys.exit(1) + +team = sys.argv[1] +project = sys.argv[2] + + +def fix_bad_time(bad_time): + # This is stupid, but time.strptime doesn't support %z in 2.6 + #return "%s-%s-%sT%sZ%s:%s" % (bad_time[:4], bad_time[5:7], bad_time[8:10], + # bad_time[11:19], bad_time[20:23], bad_time[23:]) + return "%s-%s-%sT%sZ" % (bad_time[:4], bad_time[5:7], bad_time[8:10], + bad_time[11:19]) + +# TODO: Fetch the files from the internets +issues = [] + +for issue_state in ("open", "closed"): + full_url = "http://github.com/api/v2/json/issues/list/%s/%s/%s" % (team, + project, issue_state) + with closing(urllib2.urlopen(full_url)) as issue_json: + these_issues = simplejson.load(issue_json) + issues.extend(these_issues['issues']) + + +outfile_name = "%s_%s_lp_bugs.xml" % (team, project) +bugs_outfile = codecs.open(outfile_name, "w", "utf-8-sig") +bugs_outfile.write(""" + +""") + +for issue in issues: + issue['body'] = escape(issue['body']) + issue['title'] = escape(issue['title']) + issue['lower_user'] = issue['user'].lower() + if issue['state'] == "open": + issue['status'] = "CONFIRMED" + else: + issue['status'] = "FIXRELEASED" + for bad_time in ('updated_at', 'created_at'): + issue[bad_time] = fix_bad_time(issue[bad_time]) + + bugs_outfile.write(""" + + %(created_at)s + %(title)s + + %(body)s + %(user)s + %(status)s + HIGH + + """ % issue) + + if len(issue['labels']) > 0: + bugs_outfile.write("\n") + for label in issue['labels']: + bugs_outfile.write("%s\n" % label.lower()) + bugs_outfile.write("\n") + + bugs_outfile.write(""" + + %(user)s + %(created_at)s + %(title)s + %(body)s + + """ % issue) + issue['comments'] = [] + full_url = "http://github.com/api/v2/json/issues/comments/%s/%s/%s" % \ + (team, project, issue['number']) + # github ratelimits v2 api to 60 calls per minute + time.sleep(1) + print full_url + with closing(urllib2.urlopen(full_url)) as comments_json: + try: + comments = simplejson.load(comments_json) + issue['comments'] = comments['comments'] + except: + issue['comments'] = [] + for comment in issue['comments']: + for bad_time in ('updated_at', 'created_at'): + comment[bad_time] = fix_bad_time(comment[bad_time]) + comment['body'] = escape(comment['body']) + comment['lower_user'] = comment['user'].lower() + try: + bugs_outfile.write(""" + + %(user)s + %(created_at)s + %(body)s + """ % comment) + except: + print comment + sys.exit(1) + bugs_outfile.write("\n\n") + +bugs_outfile.write("\n\n") +bugs_outfile.close() + +os.system("rnv bug-export.rnc %s" % outfile_name)