Add header to output files and allow date ranges
This change adds a brief one line header to all default output files to indicate when the analysis was generated. It also allows the gitdm configuration file to use a date of the format YYYY-MM-DD to specify the start or end date of a range of commits, rather than only allowing a git changeset hash. This is valuable for analyzing projects which do not routinely or consistently use tags, and when it is difficult to precisely pinpoint the git commit that corresponds to a time-based release. Change-Id: I899f6c6b046462addd3235e7797a3e1a63c8776c
This commit is contained in:
Dan Stangel
parent
87c9a9e333
commit
e3252c037f
58
do-it.sh
58
do-it.sh
|
|
@ -8,6 +8,13 @@ function DEBUG() {
|
|||
# enable/disable debugging output
|
||||
GITDM_DEBUG=${GITDM_DEBUG:-"off"}
|
||||
|
||||
# determine if a given parameter is a date matching the format YYYY-MM-DD,
|
||||
# i.e. 2013-09-13 This is used to decide if git should specify a start
|
||||
# date with '--since YYYY-MM-DD' rather than use an absolute changeset id
|
||||
function IS_DATE() {
|
||||
[[ $1 =~ ^[0-9][0-9][0-9][0-9]\-[0-9][0-9]\-[0-9][0-9]$ ]]
|
||||
}
|
||||
|
||||
GITBASE=${GITBASE:-~/git/openstack}
|
||||
RELEASE=${RELEASE:-havana}
|
||||
BASEDIR=$(pwd)
|
||||
|
|
@ -23,6 +30,9 @@ LP_STATS=${LP_STATS:-n}
|
|||
QUERY_LP=${QUERY_LP:-y}
|
||||
GERRIT_STATS=${GERRIT_STATS:-y}
|
||||
REMOVE_TEMPDIR=${REMOVE_TEMPDIR:-y}
|
||||
TIMESTAMP=`date`
|
||||
# brief header to prepend to all of the analysis results
|
||||
OUTPUT_HEADER="Statistics generated at ${TIMESTAMP}"
|
||||
|
||||
if [ ! -d .venv ]; then
|
||||
echo "Creating a virtualenv"
|
||||
|
|
@ -53,6 +63,13 @@ if [ "$GIT_STATS" = "y" ] ; then
|
|||
while read project revisions excludes x; do
|
||||
DEBUG "Generating git commit log for ${project}"
|
||||
cd ${GITBASE}/${project}
|
||||
# match possible dates of the format YYYY-MM-DD to use in
|
||||
# supplying git with a '--since DATE' paramter instead of a
|
||||
# range of changeset ids
|
||||
if IS_DATE $revisions; then
|
||||
DEBUG "Matched a git --since date of '${revisions}'"
|
||||
revisions="--since ${revisions}"
|
||||
fi
|
||||
git log ${GITLOGARGS} ${revisions} > "${TEMPDIR}/${project}-commits.log"
|
||||
if [ -n "$excludes" ]; then
|
||||
awk "/^commit /{ok=1} /^commit ${excludes}/{ok=0} {if(ok) {print}}" \
|
||||
|
|
@ -66,9 +83,11 @@ if [ "$GIT_STATS" = "y" ] ; then
|
|||
grep -v '^#' ${CONFIGDIR}/${RELEASE} |
|
||||
while read project x; do
|
||||
DEBUG "Generating git stats for ${project}"
|
||||
python gitdm -l 20 -n < "${TEMPDIR}/${project}-commits.log" > "${TEMPDIR}/${project}-git-stats.txt"
|
||||
echo "${OUTPUT_HEADER}" > "${TEMPDIR}/${project}-git-stats.txt"
|
||||
python gitdm -l 20 -n < "${TEMPDIR}/${project}-commits.log" >> "${TEMPDIR}/${project}-git-stats.txt"
|
||||
# also create a full dump with csv for further downstream processing
|
||||
python gitdm -n -y -z -x "${TEMPDIR}/${project}-git-stats.csv" < "${TEMPDIR}/${project}-commits.log" > "${TEMPDIR}/${project}-git-stats-all.txt"
|
||||
echo "${OUTPUT_HEADER}" > "${TEMPDIR}/${project}-git-stats.csv"
|
||||
python gitdm -n -y -z -x "${TEMPDIR}/${project}-git-stats.csv" < "${TEMPDIR}/${project}-commits.log" >> "${TEMPDIR}/${project}-git-stats-all.txt"
|
||||
done
|
||||
|
||||
DEBUG "Generating aggregate git stats for all projects"
|
||||
|
|
@ -76,7 +95,8 @@ if [ "$GIT_STATS" = "y" ] ; then
|
|||
while read project x; do
|
||||
cat "${TEMPDIR}/${project}-commits.log" >> "${TEMPDIR}/git-commits.log"
|
||||
done
|
||||
python gitdm -n -y -z -x "${TEMPDIR}/git-stats.csv" < "${TEMPDIR}/git-commits.log" > "${TEMPDIR}/git-stats.txt"
|
||||
echo "${OUTPUT_HEADER}" > "${TEMPDIR}/git-stats.txt"
|
||||
python gitdm -n -y -z -x "${TEMPDIR}/git-stats.csv" < "${TEMPDIR}/git-commits.log" >> "${TEMPDIR}/git-stats.txt"
|
||||
fi
|
||||
|
||||
if [ "$LP_STATS" = "y" ] ; then
|
||||
|
|
@ -97,11 +117,12 @@ if [ "$LP_STATS" = "y" ] ; then
|
|||
|
||||
echo "Generating launchpad statistics"
|
||||
cd ${BASEDIR}
|
||||
echo "${OUTPUT_HEADER}" > "${TEMPDIR}/${project}-lp-stats.txt"
|
||||
grep -v '^#' ${CONFIGDIR}/${RELEASE} |
|
||||
while read project x; do
|
||||
DEBUG "Generating launchpad stats for ${project}"
|
||||
grep -v '<unknown>' "${TEMPDIR}/${project}-bugs.log" |
|
||||
python lpdm -l 20 > "${TEMPDIR}/${project}-lp-stats.txt"
|
||||
python lpdm -l 20 >> "${TEMPDIR}/${project}-lp-stats.txt"
|
||||
done
|
||||
|
||||
DEBUG "Generating aggregate launchpad stats for all projects"
|
||||
|
|
@ -110,8 +131,9 @@ if [ "$LP_STATS" = "y" ] ; then
|
|||
while read project x; do
|
||||
grep -v '<unknown>' "${TEMPDIR}/${project}-bugs.log" >> "${TEMPDIR}/lp-bugs.log"
|
||||
done
|
||||
echo "${OUTPUT_HEADER}" > "${TEMPDIR}/lp-stats.txt"
|
||||
grep -v '<unknown>' "${TEMPDIR}/lp-bugs.log" |
|
||||
python lpdm -l 20 > "${TEMPDIR}/lp-stats.txt"
|
||||
python lpdm -l 20 >> "${TEMPDIR}/lp-stats.txt"
|
||||
fi
|
||||
|
||||
if [ "$GERRIT_STATS" = "y" ] ; then
|
||||
|
|
@ -119,6 +141,13 @@ if [ "$GERRIT_STATS" = "y" ] ; then
|
|||
grep -v '^#' ${CONFIGDIR}/${RELEASE} |
|
||||
while read project revisions x; do
|
||||
cd "${GITBASE}/${project}"
|
||||
# match possible dates of the format YYYY-MM-DD to use in
|
||||
# supplying git with a '--since DATE' paramter instead of a
|
||||
# range of changeset ids
|
||||
if IS_DATE $revisions; then
|
||||
DEBUG "Matched a git --since date of '${revisions}'"
|
||||
revisions="--since ${revisions}"
|
||||
fi
|
||||
git log ${revisions} |
|
||||
awk '/^ Change-Id: / { print $2 }' |
|
||||
split -l 100 -d - "${TEMPDIR}/${project}-${RELEASE}-change-ids-"
|
||||
|
|
@ -142,6 +171,13 @@ if [ "$GERRIT_STATS" = "y" ] ; then
|
|||
while read project revisions x; do
|
||||
DEBUG "Generating a list of commit IDs for ${project}"
|
||||
cd "${GITBASE}/${project}"
|
||||
# match possible dates of the format YYYY-MM-DD to use in
|
||||
# supplying git with a '--since DATE' paramter instead of a
|
||||
# range of changeset ids
|
||||
if IS_DATE $revisions; then
|
||||
DEBUG "Matched a git --since date of '${revisions}'"
|
||||
revisions="--since ${revisions}"
|
||||
fi
|
||||
git log --pretty=format:%H $revisions > \
|
||||
"${TEMPDIR}/${project}-${RELEASE}-commit-ids.txt"
|
||||
done
|
||||
|
|
@ -160,15 +196,17 @@ if [ "$GERRIT_STATS" = "y" ] ; then
|
|||
|
||||
echo "Generating gerrit statistics"
|
||||
cd ${BASEDIR}
|
||||
echo "${OUTPUT_HEADER}" > "${TEMPDIR}/${project}-gerrit-stats.txt"
|
||||
echo "${OUTPUT_HEADER}" > "${TEMPDIR}/${project}-gerrit-stats-all.txt"
|
||||
grep -v '^#' ${CONFIGDIR}/${RELEASE} |
|
||||
while read project x; do
|
||||
DEBUG "Generating gerrit statistics for ${project}"
|
||||
python gerritdm -l 20 \
|
||||
< "${TEMPDIR}/${project}-${RELEASE}-reviewers.txt" \
|
||||
> "${TEMPDIR}/${project}-gerrit-stats.txt"
|
||||
>> "${TEMPDIR}/${project}-gerrit-stats.txt"
|
||||
python gerritdm -z \
|
||||
< "${TEMPDIR}/${project}-${RELEASE}-reviewers.txt" \
|
||||
> "${TEMPDIR}/${project}-gerrit-stats-all.txt"
|
||||
>> "${TEMPDIR}/${project}-gerrit-stats-all.txt"
|
||||
done
|
||||
|
||||
DEBUG "Generating aggregate gerrit statistics for all projects"
|
||||
|
|
@ -177,8 +215,10 @@ if [ "$GERRIT_STATS" = "y" ] ; then
|
|||
while read project x; do
|
||||
cat "${TEMPDIR}/${project}-${RELEASE}-reviewers.txt" >> "${TEMPDIR}/gerrit-reviewers.txt"
|
||||
done
|
||||
python gerritdm -l 20 < "${TEMPDIR}/gerrit-reviewers.txt" > "${TEMPDIR}/gerrit-stats.txt"
|
||||
python gerritdm -z < "${TEMPDIR}/gerrit-reviewers.txt" > "${TEMPDIR}/gerrit-stats-all.txt"
|
||||
echo "${OUTPUT_HEADER}" > "${TEMPDIR}/gerrit-stats.txt"
|
||||
echo "${OUTPUT_HEADER}" > "${TEMPDIR}/gerrit-stats-all.txt"
|
||||
python gerritdm -l 20 < "${TEMPDIR}/gerrit-reviewers.txt" >> "${TEMPDIR}/gerrit-stats.txt"
|
||||
python gerritdm -z < "${TEMPDIR}/gerrit-reviewers.txt" >> "${TEMPDIR}/gerrit-stats-all.txt"
|
||||
fi
|
||||
|
||||
DEBUG "Cleaning up"
|
||||
|
|
|
|||
Loading…
Reference in New Issue