new command to build sql database directly from gerrit
Rather than downloading the review data in one command and then re-processing it to build a sql database in another, just move directly from gerrit to sqlite. Change-Id: I01545ac8e88f40b5b49148571d93fa16463e9624 Signed-off-by: Doug Hellmann <doug@doughellmann.com>
This commit is contained in:
parent
831abc3fde
commit
280cf02610
1
.gitignore
vendored
1
.gitignore
vendored
@ -26,3 +26,4 @@ releasenotes/build
|
||||
*.rpt*
|
||||
*.dat
|
||||
*.txt
|
||||
/queries/*.db
|
||||
|
@ -10,11 +10,18 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import itertools
|
||||
import logging
|
||||
import os.path
|
||||
import sqlite3
|
||||
|
||||
from cliff import command
|
||||
|
||||
from goal_tools.who_helped import report
|
||||
from goal_tools import foundation
|
||||
from goal_tools import gerrit
|
||||
from goal_tools import governance
|
||||
from goal_tools import organizations
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
@ -73,6 +80,9 @@ class QueryContributions(report.ContributionsReportBase):
|
||||
data = self.get_contributions(parsed_args)
|
||||
|
||||
cursor = db.cursor()
|
||||
print(SQL_INSERT)
|
||||
data = list(data)
|
||||
print('DATA[0]:', data[0])
|
||||
cursor.executemany(SQL_INSERT, data)
|
||||
else:
|
||||
cursor = db.cursor()
|
||||
@ -81,3 +91,115 @@ class QueryContributions(report.ContributionsReportBase):
|
||||
cursor.execute(parsed_args.query)
|
||||
col_names = (info[0] for info in cursor.description)
|
||||
return (col_names, cursor.fetchall())
|
||||
|
||||
|
||||
class DBCreate(command.Command):
|
||||
"Build a local database of contributions."
|
||||
|
||||
def get_parser(self, prog_name):
|
||||
parser = super().get_parser(prog_name)
|
||||
parser.add_argument(
|
||||
'--governance-project-list',
|
||||
default=governance.PROJECTS_LIST,
|
||||
help='location of governance project list',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--include-unofficial',
|
||||
default=False,
|
||||
action='store_true',
|
||||
help='include projects not under governance in the output',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--include-plus-one',
|
||||
default=False,
|
||||
action='store_true',
|
||||
help='include +1 votes',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--force',
|
||||
default=False,
|
||||
action='store_true',
|
||||
help='force recreating the database',
|
||||
)
|
||||
parser.add_argument(
|
||||
'query_string',
|
||||
help='gerrit query string',
|
||||
)
|
||||
parser.add_argument(
|
||||
'db_file',
|
||||
help='database to create',
|
||||
)
|
||||
return parser
|
||||
|
||||
def take_action(self, parsed_args):
|
||||
team_data = governance.Governance(
|
||||
url=parsed_args.governance_project_list)
|
||||
|
||||
cache = self.app._load_cache_file(preload=False)
|
||||
factory = gerrit.ReviewFactory(cache)
|
||||
member_factory = foundation.MemberFactory(cache)
|
||||
canonical_orgs = organizations.Organizations()
|
||||
|
||||
if os.path.exists(parsed_args.db_file):
|
||||
if not parsed_args.force:
|
||||
print('ERROR: {} already exists. '
|
||||
'Use the --force flag to overwrite.'.format(
|
||||
parsed_args.db_file))
|
||||
return 1
|
||||
else:
|
||||
os.unlink(parsed_args.db_file)
|
||||
|
||||
db = sqlite3.connect(parsed_args.db_file)
|
||||
db.execute(SQL_CREATE)
|
||||
|
||||
def get_data():
|
||||
review_source = factory.query(parsed_args.query_string)
|
||||
for review in review_source:
|
||||
|
||||
team_name = team_data.get_repo_owner(review.project)
|
||||
|
||||
if not parsed_args.include_unofficial and not team_name:
|
||||
LOG.debug(
|
||||
'filtered out %s based on repo governance status',
|
||||
review.project,
|
||||
)
|
||||
continue
|
||||
|
||||
if parsed_args.include_plus_one:
|
||||
participants = itertools.chain(
|
||||
review.participants,
|
||||
review.plus_ones,
|
||||
)
|
||||
else:
|
||||
participants = review.participants
|
||||
|
||||
for participant in participants:
|
||||
# Figure out which organization the user was
|
||||
# affiliated with at the time of the work.
|
||||
organization = None
|
||||
member = member_factory.fetch(participant.email)
|
||||
if member:
|
||||
affiliation = member.find_affiliation(participant.date)
|
||||
if affiliation and affiliation.organization:
|
||||
organization = canonical_orgs[
|
||||
affiliation.organization]
|
||||
else:
|
||||
organization = canonical_orgs.from_email(
|
||||
participant.email)
|
||||
if not organization:
|
||||
organization = "*unknown"
|
||||
|
||||
yield (review.id, review.url, review.branch,
|
||||
review.project, team_name, participant.role,
|
||||
participant.name, participant.email,
|
||||
participant.date, organization)
|
||||
|
||||
cursor = db.cursor()
|
||||
data = get_data()
|
||||
while True:
|
||||
chunk = list(itertools.islice(data, 100))
|
||||
if not chunk:
|
||||
break
|
||||
LOG.debug('inserting %d', len(chunk))
|
||||
cursor.executemany(SQL_INSERT, chunk)
|
||||
db.commit()
|
||||
|
@ -39,6 +39,7 @@ who_helped =
|
||||
contributions distinct = goal_tools.who_helped.distinct:DistinctContributions
|
||||
contributions query = goal_tools.who_helped.sql:QueryContributions
|
||||
contributions matrix = goal_tools.who_helped.matrix:MatrixContributions
|
||||
database create = goal_tools.who_helped.sql:DBCreate
|
||||
member show = goal_tools.who_helped.members:ShowMember
|
||||
changes query = goal_tools.who_helped.changes:QueryChanges
|
||||
cache remove = goal_tools.who_helped.cache:CacheRemove
|
||||
|
16
tools/build_db.sh
Executable file
16
tools/build_db.sh
Executable file
@ -0,0 +1,16 @@
|
||||
#!/bin/bash -xe
|
||||
|
||||
if [ ! -f .tox/venv/bin/activate ]; then
|
||||
tox -e venv --notest
|
||||
fi
|
||||
source .tox/venv/bin/activate
|
||||
|
||||
for input in $*
|
||||
do
|
||||
db_file=${input%.qry}.db
|
||||
|
||||
who-helped -v --debug database create \
|
||||
--force \
|
||||
"$(cat $input)" \
|
||||
"$db_file"
|
||||
done
|
Loading…
x
Reference in New Issue
Block a user