diff --git a/pipelines/monolithic.Jenkinsfile b/pipelines/monolithic.Jenkinsfile index e6dda99..600125f 100644 --- a/pipelines/monolithic.Jenkinsfile +++ b/pipelines/monolithic.Jenkinsfile @@ -200,6 +200,23 @@ pipeline { string ( name: 'JENKINS_SCRIPTS_BRANCH' ) + text ( + name: 'PATCH_LIST', + description: '''\ +
List of Gerrit URLs to apply before running the build, one per line "[PATH] URL REF", eg:
+
+  https://review.opendev.org/starlingx/config     refs/changes/71/859571/4
+  https://review.opendev.org/starlingx/stx-puppet refs/changes/75/859575/1
+  https://review.opendev.org/starlingx/tools      refs/changes/76/859576/2
+
+or with paths relative to repo root:
+
+  cgcs-root/stx/config     https://review.opendev.org/starlingx/config     refs/changes/71/859571/4
+  cgcs-root/stx/stx-puppet https://review.opendev.org/starlingx/stx-puppet refs/changes/75/859575/1
+  stx-tools                https://review.opendev.org/starlingx/tools      refs/changes/76/859576/2
+
+ ''' + ) } stages { diff --git a/pipelines/parts/clone-source.Jenkinsfile b/pipelines/parts/clone-source.Jenkinsfile index 985f7c8..cd4b6ac 100644 --- a/pipelines/parts/clone-source.Jenkinsfile +++ b/pipelines/parts/clone-source.Jenkinsfile @@ -43,6 +43,9 @@ pipeline { string ( name: 'REFRESH_SOURCE' ) + text ( + name: 'PATCH_LIST' + ) } stages { stage ("clone-source") { diff --git a/scripts/clone-source.sh b/scripts/clone-source.sh index 3ca2366..b956019 100755 --- a/scripts/clone-source.sh +++ b/scripts/clone-source.sh @@ -31,43 +31,212 @@ ln -sfn "$WORKSPACE_ROOT_SUBDIR" "$WORKSPACE_ROOT" shell() { if [[ "$1" == "--dry-run" ]] ; then echo ">>> (dry) running:" >&2 - echo "$2" >&2 + echo "$2" | sed -r 's#^#\t#' >&2 return fi echo ">>> running" >&2 - echo "$1" >&2 + echo "$1" | sed -r 's#^#\t#' >&2 ( eval "$1" ; ) } -# clone sources cd "$REPO_ROOT" -if [[ -f ".repo-init-done" ]] && ! $REFRESH_SOURCE ; then - notice "repo already initialized, exiting" - exit 0 -fi -if $DRY_RUN && [[ -f ".repo-init-done" ]] ; then - dry_run_arg="--dry-run" +# If repo doesn't exist or REFRESH_SOURCE==true: discard all local changes +# and re-pull all repos +if [[ ! -d ".repo" || ! -f ".repo-init-done" ]] || $REFRESH_SOURCE ; then + notice "Cloning sources" + + if [[ ! -d ".repo" || ! -f ".repo-init-done" ]] ; then + dry_run=false + dry_run_arg= + else + dry_run=$DRY_RUN + dry_run_arg=$DRY_RUN_ARG + fi + + # Don't change this file name: the builder container relies on it! + $dry_run || rm -f ".repo-init-done" + + # clone manifest + with_default_retries shell $dry_run_arg "repo init -u \"$MANIFEST_URL\" -b \"$MANIFEST_BRANCH\" -m \"$MANIFEST\"" + + # clean up gits: abort in-progress "rebase" & "am" commands & + # discard modified files + for d in $(repo forall -c 'echo $REPO_PATH' 2>/dev/null) ; do + [[ -d "$d" ]] || continue + shell $dry_run_arg "\ +set -e ; +cd \"$d\" +git rebase --abort >/dev/null 2>&1 || : +git am --abort >/dev/null 2>&1 || : +git cherry-pick --abort >/dev/null 2>&1 || : +git clean -d -f +git checkout . +" + done + # pull gits + with_default_retries shell $dry_run_arg "repo sync --force-sync --force-remove-dirty -j4" + touch ".repo-init-done" else - dry_run_arg= + info "repo already cloned" fi -# We can't dry run, since we need the sources -dry_run_arg= +# Apply patches in "$PATCH_LIST" parameter given one per line: +# [PATH] URL REF +if [[ -n "$PATCH_LIST" ]] ; then + notice "Applying patches" -shell $dry_run_arg "repo init -u \"$MANIFEST_URL\" -b \"$MANIFEST_BRANCH\" -m \"$MANIFEST\"" -for d in $(repo forall -c 'echo $REPO_PATH' 2>/dev/null) ; do - [[ -d "$d" ]] || continue - shell $dry_run_arg " - set -e ; - cd \"$d\" - git rebase --abort >/dev/null 2>&1 || : - git am --abort >/dev/null 2>&1 || : - git clean -d -f - git checkout . - " -done -with_default_retries shell $dry_run_arg "repo sync --force-sync --force-remove-dirty -j4" -# prevent "stx build prepare" from doing another "repo sync" -shell $dry_run_arg "touch .repo-init-done" + ALL_PROJECTS="$(repo forall -c 'echo $REPO_PROJECT $REPO_PATH')" + # Helper func to determine whether a patch URL refers to the manifest repo + is_manifest_url() { + local path="$1" + local url="$2" + [[ -z "$path" ]] || path="$(realpath -m -s "$path" --relative-to=.)" || return 1 + + # path given + if [[ "$path" == ".repo/manifests" ]] ; then + return 0 + fi + + # url equals MANIFEST_URL exactly + if [[ "$url" == "$MANIFEST_URL" ]] ; then + return 0 + fi + + # url matches any of the manifest repo remote URLs on disk + all_urls=$( (cd .repo/manifests && git config --list | sed -r -n 's#^remote[.][^.]+[.]url=##gp' | sort -u) 2>/dev/null || :) + if [[ -n "$all_urls" ]] && in_list "$url" ${all_urls} ; then + return 0 + fi + + # give up + return 1 + } + + # Helper func to find a sub-project by path & url + # If user provided a path, look for the matching project + # Else, find project that matches URL's basename + # Prints the line "PROJECT PATH" + find_project() { + local path="$1" + local url="$2" + local spec + + # path not provided: look for project that matches the URL's basename + # with or without .git at the end + if [[ -z "$path" ]] ; then + local url_base="${url##*/}" + local url_base_norm="${url_base%.git}" + local url_base_git="${url_base_norm}".git + local fail=0 + while read spec_project spec_project_path ; do + [[ -n "$spec_project" ]] || continue + local spec_project_norm="${spec_project%.git}" + local spec_project_git="${spec_project_norm}.git" + if [[ "$spec_project_norm" == "$url_base_norm" || "$spec_project_git" == "$url_base_git" ]] ; then + if [[ -n "$path" ]] ; then + error "patch url \"$url\" matches more than one project:" \ + " ${spec_project_path}" \ + " ${path}" \ + "Please specify project path explicitly in front of the URL, eg:" \ + " ${path} ${url} ${ref}" + fail=1 + continue + fi + echo "$spec_project $spec_project_path" + return 0 + fi + done <<<"$ALL_PROJECTS"$'\n' + # Errors: return false + [[ $fail -eq 0 ]] || return 1 + + # path provided: look for project matching this path + else + path="$(realpath -m -s "$path" --relative-to=.)" || return 1 + while read spec_project spec_project_path ; do + [[ -n "$spec_project" ]] || continue + spec_project_path="$(realpath -m -s "$spec_project_path" --relative-to=.)" + if [[ "$path" == "$spec_project_path" ]] ; then + echo "$spec_project $spec_project_path" + return 0 + fi + done <<<"$ALL_PROJECTS"$'\n' + fi + + # if we reach here, we couldn't find the project + error "unable to find project matching $path $url $ref" + return 1 + } + + # All patches, each element is a string: + # PROJECT PATH URL REF + PATCH_SPECS="$( + fail=0 + while read line ; do + # skip empty lines + if [[ -z "$line" || "${line[0]}" == "#" ]] ; then + continue + fi + # split by whitespace + declare -a parts=($line) + if [[ "${#parts[@]}" -eq 2 ]] ; then + path= + url="${parts[0]}" + ref="${parts[1]}" + elif [[ "${#parts[@]}" -eq 3 ]] ; then + path="${parts[0]}" + url="${parts[1]}" + ref="${parts[2]}" + else + error "Invalid patch spec \"$line\"" \ + "Expecting [PATH] URL REF" + fail=1 + continue + fi + + # manifest ? + if is_manifest_url "$path" "$url" ; then + project="-" + path=".repo/manifests" + else + # guess project and/or path + tmp="$(find_project "$path" "$url")" || return 1 + read project path <<<"$tmp" || return 1 + fi + + echo "$project $path $url $ref" + done <<<"$PATCH_LIST"$'\n' + [[ $fail -eq 0 ]] || exit 1 + )" || exit 1 + + # Reset each project that requires patching. We need to do this + # first in case PATCH_LIST includes multiple patches for the same repo, + # and in case we didn't run "repo sync" due to REFRESH_SOURCE=false + while read project path ; do + [[ -n "$project" ]] || continue + # abort in-progress "rebase" etc + info "" "--- $path: resetting git checkout" "" + shell $DRY_RUN_ARH "\ +set -e +cd \"$path\" +git rebase --abort >/dev/null 2>&1 || : +git am --abort >/dev/null 2>&1 || : +git cherry-pick --abort >/dev/null 2>&1 || : +git checkout . +" + # manifest: checkout "default" branch + if [[ $path == ".repo/manifests" ]] ; then + shell $DRY_RUN_ARG "cd \"$path\" && git checkout default" + # Other projects: checkout whatever manifest points to + else + shell $DRY_RUN_ARG "cd \"$path\" && repo sync --no-manifest-update --local-only --detach $project" + fi + done <<<"$(echo "$PATCH_SPECS" | awk '{print $1, $2}' | sort -u)" + + # Apply patches one at a time + while read project path url ref ; do + info "" "--- $path: applying patch $url $ref" "" + shell $DRY_RUN_ARG "cd \"$path\" && git fetch \"$url\" \"$ref\" && git cherry-pick FETCH_HEAD" + done <<<"$(echo "$PATCH_SPECS")" +fi