diff --git a/build-tools/branching/create_branches_and_tags.sh b/build-tools/branching/create_branches_and_tags.sh index d87e0987..27c820c4 100755 --- a/build-tools/branching/create_branches_and_tags.sh +++ b/build-tools/branching/create_branches_and_tags.sh @@ -24,7 +24,7 @@ CREATE_BRANCHES_AND_TAGS_SH_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" source "${CREATE_BRANCHES_AND_TAGS_SH_DIR}/../git-repo-utils.sh" usage () { - echo "create_branches_and_tags.sh --branch= [--tag=] [ --remotes= ] [ --projects= ] [ --manifest [ --lock-down ]]" + echo "create_branches_and_tags.sh --branch= [--tag=] [ --remotes= ] [ --projects= ] [ --gitreview-default ] [ --manifest [ --lock-down | --soft-lock-down ] [ --default-revision ]]" echo "" echo "Create a branch and a tag in all listed projects, and all" echo "projects hosted by all listed remotes. Lists are comma separated." @@ -33,13 +33,19 @@ usage () { echo "If the tag is omitted, one is automativally generate by adding the" echo "prefix 'v' to the branch name." echo "" - echo "If a manifest is requested, it will recieve the name '.xml' and" - echo "it will specify the branch as the revision for all tagged projects." + echo "If a manifest is requested, the current manifest is modified." + echo "to specify the new branch for all select remotes and projects." echo "If lockdown is requested, all other projects get the current" echo "HEAD's sha set as the revision." + echo "If default-revision is selected, then the manifest default revision" + wcho "will be set." + echo "" + echo "If a gitreview-default is selected, then all branched projects" + echo "with a .gitreview file will have a defaultbranch entry added" + echo "or updated." } -TEMP=$(getopt -o h --long remotes:,projects:,branch:,tag:,manifest,lock-down,help -n 'create_branches_and_tags.sh' -- "$@") +TEMP=$(getopt -o h --long remotes:,projects:,branch:,tag:,manifest,lock-down,hard-lock-down,soft-lock-down,default-revision,gitreview-default,help -n 'create_branches_and_tags.sh' -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -49,6 +55,8 @@ eval set -- "$TEMP" HELP=0 MANIFEST=0 LOCK_DOWN=0 +GITREVIEW_DEFAULT=0 +SET_DEFAULT_REVISION=0 remotes="" projects="" branch="" @@ -59,15 +67,19 @@ repo_root_dir="" while true ; do case "$1" in - -h|--help) HELP=1 ; shift ;; - --remotes) remotes+=$(echo "$2 " | tr ',' ' '); shift 2;; - --projects) projects+=$(echo "$2 " | tr ',' ' '); shift 2;; - --branch) branch=$2; shift 2;; - --tag) tag=$2; shift 2;; - --manifest) MANIFEST=1 ; shift ;; - --lock-down) LOCK_DOWN=1 ; shift ;; - --) shift ; break ;; - *) usage; exit 1 ;; + -h|--help) HELP=1 ; shift ;; + --remotes) remotes+=$(echo "$2 " | tr ',' ' '); shift 2;; + --projects) projects+=$(echo "$2 " | tr ',' ' '); shift 2;; + --branch) branch=$2; shift 2;; + --tag) tag=$2; shift 2;; + --manifest) MANIFEST=1 ; shift ;; + --lock-down) LOCK_DOWN=2 ; shift ;; + --hard-lock-down) LOCK_DOWN=2 ; shift ;; + --soft-lock-down) LOCK_DOWN=1 ; shift ;; + --default-revision) SET_DEFAULT_REVISION=1 ; shift ;; + --gitreview-default) GITREVIEW_DEFAULT=1 ; shift ;; + --) shift ; break ;; + *) usage; exit 1 ;; esac done @@ -88,6 +100,37 @@ if [ $? -ne 0 ]; then exit 1 fi +update_gitreview () { + local DIR=$1 + ( + cd $DIR || exit 1 + if [ $GITREVIEW_DEFAULT -eq 1 ] && [ -f .gitreview ]; then + if ! grep -q "^defaultbranch=$branch$" .gitreview; then + echo "Updating defaultbranch in ${DIR}/.gitreview" + if grep -q defaultbranch= .gitreview; then + sed "s#\(defaultbranch=\).*#\1$branch#" -i .gitreview + else + echo "defaultbranch=$branch" >> .gitreview + fi + + git add .gitreview + if [ $? != 0 ] ; then + echo_stderr "ERROR: failed to add .gitreview in ${DIR}" + exit 1 + fi + + git commit -s -m "Update .gitreview for $branch" + if [ $? != 0 ] ; then + echo_stderr "ERROR: failed to commit .gitreview in ${DIR}" + exit 1 + fi + else + echo "defaultbranch in ${DIR}/.gitreview already set" + fi + fi + ) +} + if [ $MANIFEST -eq 1 ]; then manifest=$(repo_manifest $repo_root_dir) if [ $? -ne 0 ]; then @@ -205,6 +248,7 @@ for subgit in $SUBGITS; do git checkout $branch fi + # check if destination tag already exists tag_check=$(git tag -l $tag) if [ -z "$tag_check" ]; then echo "Creating tag $tag in ${subgit}" @@ -216,6 +260,8 @@ for subgit in $SUBGITS; do else echo "Tag '$tag' already exists in ${subgit}" fi + + update_gitreview ${subgit} || exit 1 ) || exit 1 done ) || exit 1 @@ -276,8 +322,10 @@ if [ $MANIFEST -eq 1 ]; then exit 1 fi + update_gitreview ${manifest_dir} || exit 1 + echo "Creating manifest ${new_manifest_name}" - manifest_set_revision "${manifest}" "${new_manifest}" "$branch" ${LOCK_DOWN} $projects || exit 1 + manifest_set_revision "${manifest}" "${new_manifest}" "$branch" ${LOCK_DOWN} ${SET_DEFAULT_REVISION} $projects || exit 1 echo "Move manifest ${new_manifest_name}, overwriting ${manifest_name}" \cp -f "${manifest}" "${manifest}.save" diff --git a/build-tools/branching/push_branches_tags.sh b/build-tools/branching/push_branches_tags.sh index 1f94cc42..baa73bb6 100755 --- a/build-tools/branching/push_branches_tags.sh +++ b/build-tools/branching/push_branches_tags.sh @@ -17,6 +17,7 @@ PUSH_BRANCHES_TAGS_SH_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" source "${PUSH_BRANCHES_TAGS_SH_DIR}/../git-repo-utils.sh" +source "${PUSH_BRANCHES_TAGS_SH_DIR}/../url_utils.sh" usage () { echo "push_branches_tags.sh --branch= [--tag=] [ --remotes= ] [ --projects= ] [ --manifest ]" @@ -151,6 +152,8 @@ for subgit in $SUBGITS; do ( cd $subgit + git fetch --all + branch_check=$(git branch -a --list $branch) if [ -z "$branch_check" ]; then echo_stderr "ERROR: Expected branch '$branch' to exist in ${subgit}" @@ -169,21 +172,49 @@ for subgit in $SUBGITS; do exit 1 fi - if [ "${review_method}" == "gerrit" ]; then - remote=$(git_repo_review_remote) - else - remote=$(git_repo_remote) + remote=$(git_remote) + if [ "${remote}" == "" ]; then + echo_stderr "ERROR: Failed to determine remote in ${manifest_dir}" + exit 1 fi - if [ "${remote}" == "" ]; then - echo_stderr "ERROR: Failed to determine remote in ${subgit}" + if [ "${review_method}" == "gerrit" ]; then + review_remote=$(git_repo_review_remote) + else + review_remote=${remote} + fi + + if [ "${review_remote}" == "" ]; then + echo_stderr "ERROR: Failed to determine review_remote in ${subgit}" exit 1 fi + branch_check=$(git branch -a --list $remote/$branch) + if [ "${branch_check}" != "" ]; then + echo "Branch $branch already exists in ${subgit}" + exit 0 + fi + echo "Pushing branch $branch in ${subgit}" if [ "${review_method}" == "gerrit" ]; then - echo "git push --tags ${remote} ${branch}" - git push --tags ${remote} ${branch} + url=$(git_repo_review_url) + if [ "${review_remote}" == "" ]; then + echo_stderr "ERROR: Failed to determine review_url in ${subgit}" + exit 1 + fi + + host=$(url_server "${url}") + port=$(url_port "${url}") + path=$(url_path "${url}") + if [ "${host}" == "review.opendev.org" ]; then + git push ${review_remote} ${tag} && \ + ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \ + git config --local --replace-all "branch.${branch}.merge" refs/heads/${branch} && \ + git review --topic="${branch}" + else + echo "git push --tags ${remote} ${branch}" + git push --tags ${remote} ${branch} + fi else echo "git push --tags --set-upstream ${remote} ${branch}" git push --tags --set-upstream ${remote} ${branch} @@ -232,23 +263,44 @@ if [ $MANIFEST -eq 1 ]; then exit 1 fi - - remote=$(git_review_remote) + remote=$(git_remote) if [ "${remote}" == "" ]; then echo_stderr "ERROR: Failed to determine remote in ${manifest_dir}" exit 1 fi + review_remote=$(git_review_remote) + if [ "${review_remote}" == "" ]; then + echo_stderr "ERROR: Failed to determine review_remote in ${manifest_dir}" + exit 1 + fi + echo "Pushing branch $branch in ${manifest_dir}" if [ "${review_method}" == "gerrit" ]; then # Is a reviewless push possible as part of creating a new branch in gerrit? - git push --tags ${remote} ${branch} + url=$(git_review_url) + if [ "${review_remote}" == "" ]; then + echo_stderr "ERROR: Failed to determine review_url in ${subgit}" + exit 1 + fi + + host=$(url_server "${url}") + port=$(url_port "${url}") + path=$(url_path "${url}") + if [ "${host}" == "review.opendev.org" ]; then + git push ${review_remote} ${tag} && \ + ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \ + git config --local --replace-all "branch.${branch}.merge" refs/heads/${branch} && \ + git review --yes --topic="${branch}" + else + git push --tags ${review_remote} ${branch} + fi else - git push --tags --set-upstream ${remote} ${branch} + git push --tags --set-upstream ${review_remote} ${branch} fi if [ $? != 0 ] ; then - echo_stderr "ERROR: Failed to push tag '${tag}' to remote '${remote}' in ${manifest_dir}" + echo_stderr "ERROR: Failed to push tag '${tag}' to remote '${review_remote}' in ${manifest_dir}" exit 1 fi ) || exit 1 diff --git a/build-tools/repo-utils.sh b/build-tools/repo-utils.sh index 9514b0e9..49f3d8eb 100644 --- a/build-tools/repo-utils.sh +++ b/build-tools/repo-utils.sh @@ -140,6 +140,50 @@ repo_is_project () { } +# +# manifest_get_revision_of_project +# +# Extract the revision of a project within the manifest. +# The default revision is supplied in the absence +# of an explicit project revision. +# +# manifest = Path to manifest. +# project-name = name of project. +# +manifest_get_revision_of_project () { + local manifest="${1}" + local project="${2}" + + local default_revision="" + local revision="" + + default_revision=$(manifest_get_default_revision "${manifest}") + revision=$(grep ' +# +# Extract the default revision of the manifest, if any. +# +# manifest = Path to manifest. +# +manifest_get_default_revision () { + local manifest="${1}" + + grep ' # @@ -149,8 +193,10 @@ repo_is_project () { # revision = A branch, tag ,or sha. Branch and SHA can be used # directly, but repo requires that a tag be in the form # "refs/tags/". -# lock_down = 0 or 1. If 1, set a revision on all other non-listed +# lock_down = 0,1 or 2. If 2, set a revision on all other non-listed # projects to equal the SHA of the current git head. +# If 1, similar to 2, but only if the project doesn't have +# some other form of revision specified. # project-list = A space seperated list of projects. Listed projects # will have their revision set to the provided revision # value. @@ -160,9 +206,11 @@ manifest_set_revision () { local new_manifest="${2}" local revision="${3}" local lock_down="${4}" - shift 4 + local set_default="${5}" + shift 5 local projects="${@}" + local old_default_revision="" local repo_root_dir="" local line="" local FOUND=0 @@ -192,11 +240,32 @@ manifest_set_revision () { return 1 fi + old_default_revision=$(manifest_get_default_revision "${old_manifest}") + if [ ${set_default} -eq 1 ] && [ "${old_default_revision}" == "" ]; then + # We only know how to alter an existing default revision, not set a + # new one, so continue without setting a default. + set_default=0 + fi + while IFS= read -r line; do echo "${line}" | grep -q '&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + if echo "$URL" | grep -q '[:][/][/]' ;then + echo "$URL" | sed 's#^\(.*\)://.*$#\1#' + else + echo "http" + fi + return 0 +} + +url_login () { + local URL="$1" + + if [ "$URL" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + echo "$URL" | sed 's#^.*://\([^/]*\)/.*$#\1#' + return 0 +} + +url_user () { + local URL="$1" + local LOGIN + + if [ "$URL" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + url_login "$URL" | sed -e '/@/! s#.*## ; s#\([^@]*\)@.*#\1#' + if [ ${PIPESTATUS[0]} -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): url_login failed" + return 1 + fi + + return 0 +} + +url_port () { + local URL="$1" + local LOGIN + + if [ "$URL" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + url_login "$URL" | sed -e '/:/! s#.*## ; s#[^:]*:\([^:]*\)#\1#' + if [ ${PIPESTATUS[0]} -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): url_login failed" + return 1 + fi + + return 0 +} + +url_server () { + local URL="$1" + local LOGIN + + if [ "$URL" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + url_login "$URL" | sed 's#^.*@## ; s#:.*$##' + if [ ${PIPESTATUS[0]} -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): url_login failed" + return 1 + fi + + return 0 +} + +url_path () { + local URL="$1" + + if [ "$URL" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + echo "$URL" | sed 's#^.*://[^/]*/\(.*\)$#\1#' + return 0 +} + +# +# url_path_to_fs_path: +# +# Convert url format path to file system format. +# e.g. replace %20 with ' '. +# +# Note: Does NOT test the output path to ensure there are +# no illegal file system characters. +# +url_path_to_fs_path () { + local INPUT_PATH="$1" + local TEMP + + if [ "$INPUT_PATH" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + # Deviate from URI spec by not substituding '+' with ' '. + # It would alias '%20' and we need unique mappings. + # TEMP="${INPUT_PATH//+/ }" + + TEMP="$INPUT_PATH" + printf '%b' "${TEMP//%/\\x}" + return 0 +} + +# +# fs_path_to_url_path: +# +# Convert file system format path to url format. +# e.g. replace ' ' with %20. +# +fs_path_to_url_path () { + local INPUT_PATH="$1" + local LENGTH + local POS + local CHAR + + if [ "$INPUT_PATH" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + LENGTH="${#INPUT_PATH}" + for (( POS = 0; POS < LENGTH; POS++ )); do + CHAR="${1:POS:1}" + case $CHAR in + [/a-zA-Z0-9.~_-]) + # Reference https://metacpan.org/pod/URI::Escape + printf "$CHAR" + ;; + *) + printf '%%%02X' "'$CHAR" + ;; + esac + done + + return 0 +} + +# +# normalize_path: +# +# 1) replace // with / +# 2) replace /./ with / +# 3) Remove trailing / +# 4) Remove leading ./ +# + +normalize_path () { + local INPUT_PATH="$1" + + if [ "$INPUT_PATH" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + echo "$INPUT_PATH" | sed 's#[/]\+#/#g ; s#[/][.][/]#/#g ; s#/$## ; s#^[.]/##' + return 0 +} + + +# +# repo_url_to_sub_path: +# +repo_url_to_sub_path () { + local URL="$1" + local FAMILY="" + local SERVER="" + local URL_PATH="" + local FS_PATH="" + + if [ "$URL" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + # set FAMILY from URL + echo $URL | grep -q 'centos[.]org' && FAMILY=centos + echo $URL | grep -q 'fedoraproject[.]org[/]pub[/]epel' && FAMILY=epel + + SERVER=$(url_server "$URL") + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): url_server '$URL'" + return 1 + fi + + URL_PATH="$(url_path "$URL")" + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): url_path '$URL'" + return 1 + fi + + FS_PATH="$(url_path_to_fs_path "$URL_PATH")" + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): url_path_to_fs_path '$URL_PATH'" + return 1 + fi + + FS_PATH="$(normalize_path "$FS_PATH")" + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): normalize_path '$FS_PATH'" + return 1 + fi + + normalize_path "./$FAMILY/$SERVER/$FS_PATH" + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): normalize_path './$FAMILY/$SERVER/$FS_PATH'" + return 1 + fi + + return 0 +} + +CENGN_PROTOCOL="http" +CENGN_HOST="mirror.starlingx.cengn.ca" +CENGN_PORT="80" +CENGN_URL_ROOT="mirror" + +url_to_stx_mirror_url () { + local URL="$1" + local DISTRO="$2" + local URL_PATH="" + local FS_PATH="" + + if [ "$URL" == "" ] || [ "$DISTRO" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + FS_PATH="$(repo_url_to_sub_path "$URL")" + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): repo_url_to_sub_path '$URL'" + return 1 + fi + + URL_PATH=$(fs_path_to_url_path "$FS_PATH") + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): fs_path_to_url_path '$FS_PATH'" + return 1 + fi + + echo "$CENGN_PROTOCOL://$CENGN_HOST:$CENGN_PORT/$CENGN_URL_ROOT/$DISTRO/$URL_PATH" + return 0 +}