// vim: syn=groovy // // Copyright (c) 2022 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // library "common@${params.JENKINS_SCRIPTS_BRANCH}" PROPS = null IMG_PARAMS = null IMAGES_FAILED = false def parseProps(text) { def x = {} for (line in text.split (/\n+/)) { if (line.matches (/\s*(?:#.*)?#/)) { continue } parts = line.split ("=", 2) key = parts[0] value = parts[1] x."${key}" = value } return x } def loadEnv() { def data = {} data.NEED_BUILD = false data.SUPPRESS_DOCKER_IMAGE_BUILD_ERRORS = params.SUPPRESS_DOCKER_IMAGE_BUILD_ERRORS ws(params.BUILD_HOME) { if (fileExists ("NEED_BUILD")) { data.NEED_BUILD = true } } final String configText = sh (script: "${Constants.SCRIPTS_DIR}/print-config.sh", returnStdout: true) final props = parseProps (configText) data.BUILD_OUTPUT_HOME_URL = props.BUILD_OUTPUT_HOME_URL data.PUBLISH_URL = props.PUBLISH_URL data.BUILD_REMOTE_CLI = props.BUILD_REMOTE_CLI == "true" PROPS = data return data.NEED_BUILD } def partJobName (name) { final String folder = env.JOB_NAME.replaceAll (/(.*\/).+$/, '$1'); if (folder == env.JOB_NAME) { error "This job must be in a Jenkins folder!" } return "/" + folder + "parts/" + name } def runPart (name, params = [], propagate = true) { // Tell Jenkins to checkout the same commit of the sub-job's Jenkinsfile, // as the current builds' Jenkinsfile's commit. final gitRef = string (name: 'JENKINS_SCRIPTS_BRANCH', value: env.GIT_COMMIT) return build ( job: partJobName (name), parameters: copyCurrentParams() + [ gitRef ] + params, propagate: propagate ) } def runImagesPart (name, params = []) { // Ignore docker image - related errors. In this case We // prevent sub-jobs from raising exceptions and failing the // current build. Instead we note when an image-related // job has failed, and skip all subsequent image-related // jobs. if (PROPS.SUPPRESS_DOCKER_IMAGE_BUILD_ERRORS) { if (!IMAGES_FAILED) { final jobName = partJobName (name) final res = runPart (name, IMG_PARAMS + params, false).result if (res == 'ABORTED') { // FIXME: make current build ABORTED here error ("child job ${jobName} aborted") } if (res == 'SUCCESS' || res == 'UNSTABLE') { return true } print ("*** ERROR: child job ${jobName} failed!") IMAGES_FAILED = true } return false } // Otherwise, just call the subjob normally - ie its failure // will propagate to the current build runPart (name, IMG_PARAMS + params) return true } def printBuildFooter() { if (PROPS) { String msg = "" msg += "\n" msg += "========================================\n" msg += "\n" if (PROPS.NEED_BUILD) { msg += "Build output: ${PROPS.BUILD_OUTPUT_HOME_URL}\n" if (PROPS.PUBLISH_URL) { msg += "Publish output: ${PROPS.PUBLISH_URL}\n" } if (IMAGES_FAILED) { msg += "\n" msg += "WARNING:\n" msg += "WARNING: docker images build attempted, but failed!\n" msg += "WARNING: see log output above\n" msg += "WARNING:\n" } } else { echo "*** NO CHANGES - BUILD NOT REQUIRED" } msg += "\n" msg += "========================================\n" msg += "\n" echo (msg) } } setBuildDescr() pipeline { agent any options { timestamps() } parameters { string ( name: 'MASTER_JOB_NAME' ) string ( name: 'MASTER_BUILD_NUMBER' ) string ( name: 'BUILD_HOME' ) string ( name: 'TIMESTAMP', ) string ( name: 'PUBLISH_TIMESTAMP' ) booleanParam ( name: 'REBUILD_BUILDER_IMAGES' ) booleanParam ( name: 'BUILDER_USE_DOCKER_CACHE' ) booleanParam ( name: 'REFRESH_SOURCE' ) booleanParam ( name: 'BUILD_PACKAGES' ) string ( name: 'BUILD_PACKAGES_LIST' ) booleanParam ( name: 'PKG_REUSE' ) booleanParam ( name: 'BUILD_ISO' ) booleanParam ( name: 'BUILD_RT' ) booleanParam ( name: 'DRY_RUN' ) booleanParam ( name: 'SHELL_XTRACE' ) booleanParam ( name: 'CLEAN_PACKAGES' ) booleanParam ( name: 'CLEAN_ISO' ) booleanParam ( name: 'CLEAN_REPOMGR' ) booleanParam ( name: 'CLEAN_DOWNLOADS' ) booleanParam ( name: 'CLEAN_DOCKER' ) booleanParam ( name: 'FORCE_BUILD' ) booleanParam ( name: 'BUILD_HELM_CHARTS' ) booleanParam ( name: 'FORCE_BUILD_WHEELS' ) booleanParam ( name: 'BUILD_DOCKER_BASE_IMAGE' ) booleanParam ( name: 'BUILD_DOCKER_IMAGES' ) string ( name: 'DOCKER_IMAGE_LIST' ) booleanParam ( name: 'PUSH_DOCKER_IMAGES' ) booleanParam ( name: 'SUPPRESS_DOCKER_IMAGE_BUILD_ERRORS', defaultValue: false ) booleanParam ( name: 'IMPORT_BUILD' ) string ( name: 'IMPORT_BUILD_DIR' ) booleanParam ( name: 'USE_DOCKER_CACHE', ) string ( name: 'JENKINS_SCRIPTS_BRANCH' ) text ( name: 'PATCH_LIST', defaultValue: '-', description: '''\
List of Gerrit URLs to apply before running the build, one per line "[PATH] URL REF", eg:

  https://review.opendev.org/starlingx/config     refs/changes/71/859571/4
  https://review.opendev.org/starlingx/stx-puppet refs/changes/75/859575/1
  https://review.opendev.org/starlingx/tools      refs/changes/76/859576/2

or with paths relative to repo root:

  cgcs-root/stx/config     https://review.opendev.org/starlingx/config     refs/changes/71/859571/4
  cgcs-root/stx/stx-puppet https://review.opendev.org/starlingx/stx-puppet refs/changes/75/859575/1
  stx-tools                https://review.opendev.org/starlingx/tools      refs/changes/76/859576/2
''' ) } stages { stage('INIT') { steps { script { // Initialize BUILD_HOME, create build.conf & stx.conf runPart ("init-env") // Update source tree runPart ("clone-source") // create BUILD & stx.conf runPart ("configure-build") // Stop containers before updating source treee runPart ("stop-containers") // Create chnagelog, LAST_COMMITS, NEED_BUILD etc runPart ("create-changelog") // Is build required? if (loadEnv()) { IMG_PARAMS = [ string (name: 'BUILD_STREAM', value: 'stable') ] } else { println "*** NO CHANGES, BUILD NOT REQUIRED ***" } } } } // This stage runs only if build is required stage('X0') { when { expression { PROPS.NEED_BUILD } } stages { stage('PREPARE') { steps { // Login to host's docker runPart ("host-docker-login") // Delete or keep packages, aptly state, etc depending on build params runPart ("clean-build") // start containers runPart ("build-env-containers") runPart ("start-containers") // login to docker early to catch login errors runPart ("docker-login") } } // populate mirror/ stage('DOWNLOAD') { steps { runPart ("download-prerequisites") } } // build packages stage('PACKAGES') { when { expression { params.BUILD_PACKAGES } } steps { runPart ("build-packages") runPart ("publish-packages") } } // Generate initial helm charts. We will re-generate them after // building docker images, if requested in order to replace // image tags by locally-built images stage('HELM:initial') { when { expression { params.BUILD_HELM_CHARTS } } steps { runPart ("build-helm-charts", IMG_PARAMS) runPart ("publish-helm-charts", IMG_PARAMS) } } // Build ISO & images in parallel stage('X1') { parallel { stage('ISO') { when { expression { params.BUILD_ISO } } steps { script { runPart ("build-iso") runPart ("publish-iso") sh ("BUILD_STATUS=success ${Constants.SCRIPTS_DIR}/create-latest-iso-symlinks.sh") } } } // stage('ISO') stage('IMAGES') { when { expression { params.BUILD_DOCKER_BASE_IMAGE || params.BUILD_DOCKER_IMAGES } } stages { stage('IMAGES:base') { when { expression { ! IMAGES_FAILED && params.BUILD_DOCKER_BASE_IMAGE } } steps { script { runImagesPart ("build-docker-base") } } } stage('IMAGES:wheels') { when { expression { ! IMAGES_FAILED && params.BUILD_DOCKER_IMAGES } } steps { script { runImagesPart ("build-wheels") runImagesPart ("publish-wheels") } } } stage('IMAGES:images') { when { expression { ! IMAGES_FAILED && params.BUILD_DOCKER_IMAGES } } steps { script { runImagesPart ("build-docker-images") } } } stage('IMAGES:helm') { // Rebuild helm charts even if image builds failed. // This will record any images that were built sucessfully in the helm charts when { expression { params.BUILD_DOCKER_IMAGES && params.BUILD_HELM_CHARTS } } steps { script { runPart ("build-helm-charts", IMG_PARAMS) runPart ("publish-helm-charts", IMG_PARAMS) } } } stage('IMAGES:symlinks') { // Create the symlink even if some images failed. // FIXME: remove all logic re publishing failed docker builds // once all images have been fixed for Debian // when { expression { ! IMAGES_FAILED } } steps { script { // copy image lists to publish root and create the "latest_docker_image_build" symlinks // in publish and archive roots sh ("BUILD_STATUS=success ${Constants.SCRIPTS_DIR}/create-latest-containers-symlinks.sh") } } } } // stages } // stage('IMAGES') stage('remote-cli') { when { expression { PROPS.BUILD_REMOTE_CLI } } steps { runPart ("build-remote-cli") } } stage('export-dir') { steps { runPart ("build-export-dir") } } } }// stage('X1') } // stages post { always { echo "build result: ${currentBuild.result}" runPart ("stop-containers") runPart ("archive-misc") // archive anything we may have missed saveCurrentJenkinsBuildInfo() // save this job's build number on disk (for publish-logs) } success { // copy LAST_COMMITS to archive root & update the "latest_build" symlink in // both archive and publish roots sh ("BUILD_STATUS=success ${Constants.SCRIPTS_DIR}/create-latest-symlinks.sh") printBuildFooter() // Print archive & publish URLs runPart ("publish-logs") // publish this job's Jenkins log } unsuccessful { sh ("BUILD_STATUS=fail ${Constants.SCRIPTS_DIR}/create-latest-symlinks.sh") runPart ("publish-logs") // publish this job's Jenkins log } } } // stage X0 } // stages }