From 65ffa5aff182347fcf80b3a82a73c2b2fd554cb5 Mon Sep 17 00:00:00 2001 From: Dimitrios Markou Date: Tue, 2 Apr 2019 14:26:14 -0500 Subject: [PATCH] Pipeline for Airsloop site Change-Id: I407f833ff935682f74252ddb69eab9f19f248d9d Signed-off-by: Dimitrios Markou --- tools/gate/airsloop/Jenkinsfile | 371 ++++++++++++++++++++++++++++++++ tools/gate/airsloop/seed.groovy | 87 ++++++++ 2 files changed, 458 insertions(+) create mode 100644 tools/gate/airsloop/Jenkinsfile create mode 100644 tools/gate/airsloop/seed.groovy diff --git a/tools/gate/airsloop/Jenkinsfile b/tools/gate/airsloop/Jenkinsfile new file mode 100644 index 000000000..9be69c864 --- /dev/null +++ b/tools/gate/airsloop/Jenkinsfile @@ -0,0 +1,371 @@ +// shared libaries used within the pipeline +// https://github.com/att-comdev/cicd/blob/master/vars + + +import org.yaml.snakeyaml.Yaml +import org.yaml.snakeyaml.DumperOptions; + +import groovy.json.JsonSlurperClassic +import groovy.json.JsonOutput + +PEGLEG_IMAGE = 'quay.io/airshipit/pegleg:178c058474fb632806e281673d3eaf6be80fa854' + +KEYSTONE_URL = 'http://keystone.ucp.svc.cluster.local' +SHIPYARD_URL = 'http://shipyard-api.ucp.svc.cluster.local/api/v1.0' + +uuid = UUID.randomUUID().toString() + +SITE_NAME='airsloop' +IPMI_CREDS = 'airsloop-ipmi' + +GENESIS_IP = '10.22.71.21' +GENESIS_CREDS = 'airsloop-key' + +GENESIS_IPMI_IP = '10.22.104.21' + +IPMI_IPS = ['10.22.104.22'] + +AIRSHIP_MANIFESTS_REPO = 'https://review.opendev.org/airship/treasuremap' +DISPLAY_NAME = 'manual' + +if (env.GERRIT_REFSPEC) { + AIRSHIP_MANIFESTS_REF = GERRIT_REFSPEC + DISPLAY_NAME = GERRIT_EVENT_TYPE +} else if (AIRSHIP_MANIFESTS_REF == 'uplift') { + DISPLAY_NAME = 'uplift' +} + +ARTF_BASE = "cicd/logs/${JOB_NAME}/${BUILD_ID}" + +currentBuild.displayName = "#${BUILD_NUMBER} ${DISPLAY_NAME}" + + +//// git utils + +def clone(String ref){ + + def refspec = '' + + // override refspec if patchset provided + if (ref.contains('refs')) { + refspec = "${ref}:${ref}" + } + + // base uplift on latest master + if (ref == 'uplift') { + ref = 'master' + } + + checkout poll: false, + scm: [$class: 'GitSCM', + branches: [[name: ref]], + doGenerateSubmoduleConfigurations: false, + extensions: [[$class: 'CleanBeforeCheckout']], + submoduleCfg: [], + userRemoteConfigs: [[refspec: refspec, + url: AIRSHIP_MANIFESTS_REPO ]]] +} + + + +//// bare-metal utils + +def reset_bare_metal = { + stage ('Bare-Metal Reset') { + + withCredentials([usernamePassword(credentialsId: IPMI_CREDS, + usernameVariable: 'IUSER', + passwordVariable: 'IPASS')]) { + + def auth = redfish.getBasicAuth(IUSER, IPASS) + + // power cycle genesis + redfish.powerOff(GENESIS_IPMI_IP, auth) + redfish.powerOn(GENESIS_IPMI_IP, auth) + + // shutdown all other nodes + IPMI_IPS.each() { + redfish.powerOff(it, auth) + } + } + + } +} + + +//// manifest utils + +def resolve_versions = { + + def versions = readYaml file: 'global/software/config/versions.yaml' + PROMENADE_IMAGE = versions.data.images.ucp.promenade.promenade + + def shipyard = readYaml file: "site/${SITE_NAME}/secrets/passphrases/ucp_shipyard_keystone_password.yaml" + SHIPYARD_PASSWD = shipyard.data +} + +def pegleg_site_collect = { + stage('Pegleg Site Collect') { + + configFileProvider([configFile(fileId: 'airsloop-site-definition', + targetLocation: "site/${SITE_NAME}/site-definition.yaml")]) { + } + + withCredentials([sshUserPrivateKey(credentialsId: 'jenkins-attcomdev-key', + keyFileVariable: 'SSH_KEY', + usernameVariable: 'SSH_USER')]) { + + sh "cp ${SSH_KEY} ssh-key" + + auth = "-u ${SSH_USER} -k /target/ssh-key" + cmd = "pegleg site ${auth} -r /target collect ${SITE_NAME} -s /target/${SITE_NAME}" + sh "sudo docker run --rm -t -v \$(pwd):/target ${PEGLEG_IMAGE} ${cmd}" + } + + sh "tar czf site-config.tar.gz ${SITE_NAME}" + + archiveArtifacts 'site-config.tar.gz' + } +} + +def prom_config_gen = { + stage ("Promenade Config Gen") { + + sh "mkdir -p promenade-bundle" + + opts = '--rm -t -w /target -v $(pwd):/target' + cmd = "promenade build-all --validators -o promenade-bundle ${SITE_NAME}/*.yaml" + sh "sudo docker run ${opts} ${PROMENADE_IMAGE} ${cmd}" + + sh 'tar czf promenade-bundle.tar.gz promenade-bundle' + archiveArtifacts 'promenade-bundle.tar.gz' + } +} + + +//// genesis utils + +def genesis_cleanup = { + stage('Genesis Cleanup') { + + dfiles = ['promenade', + 'promenade-bundle', + 'promenade-bundle.tar.gz', + SITE_NAME, + 'airsloop.tar.gz', + 'debug-cab23-r720-11.tgz', + '/var/lib/docker', + '/var/lib/kubelet'] + + dfiles.each() { + ssh.cmd (GENESIS_CREDS, GENESIS_IP, "sudo rm -rf ${it}") + } + + ssh.cmd (GENESIS_CREDS, GENESIS_IP, + 'git clone https://opendev.org/airship/promenade') + ssh.cmd (GENESIS_CREDS, GENESIS_IP, + 'sudo -S promenade/tools/cleanup.sh -f') + + } +} + +def debug_report = { + ssh.cmd (GENESIS_CREDS, GENESIS_IP, 'sudo debug-report.sh') + ssh.get (GENESIS_CREDS, GENESIS_IP, 'debug-airsloop-control-1.tgz', '.') + + archiveArtifacts 'debug-airsloop-control-1.tgz' + + artifactory.upload("debug-airsloop-control-1.tgz", + "${ARTF_BASE}/debug-airsloop-control-1.tgz") +} + +def genesis_deploy = { + stage('Genesis Deploy') { + + try { + + ssh.put(GENESIS_CREDS, GENESIS_IP, 'promenade-bundle.tar.gz', '.') + ssh.cmd(GENESIS_CREDS, GENESIS_IP, 'tar xvzf promenade-bundle.tar.gz') + + timeout (90) { + ssh.cmd(GENESIS_CREDS, GENESIS_IP, + 'sudo promenade-bundle/genesis.sh') + + // fixme: there is notable initial slowness likely due to coredns + // going out of service and taking time to recover + // this is a long time issue and needs to be taken look at + retry(2) { + ssh.cmd(GENESIS_CREDS, GENESIS_IP, + 'sudo -S promenade-bundle/validate-genesis.sh') + } + } + + sleep 120 // wait for k8s to calm down + + } catch (err) { + debug_report() + error(err) + } + } +} + +def shipyard_deploy = { action -> + try { + def req = keystone.retrieveToken(SHIPYARD_PASSWD, KEYSTONE_URL, false) + def token = req.getHeaders()["X-Subject-Token"][0] + shipyard2.uploadConfig(uuid, token, SHIPYARD_URL, SITE_NAME) + shipyard2.waitAction(action, uuid, SHIPYARD_URL, SHIPYARD_PASSWD, KEYSTONE_URL, false) + + } catch (err) { + debug_report() + error(err) + } + + ssh.cmd(GENESIS_CREDS, GENESIS_IP, + 'sudo kubectl get pods --all-namespaces -o wide -a=false') +} + + +//// uplift utils + +def uplift_versions = { + stage('Uplift Versions') { + sh 'sudo apt-get install python3-yaml python3-git -y' + + def cmd = 'tools/updater.py --in-file global/software/config/versions.yaml' + + if (!UPLIFT_BLACKLIST.isEmpty()) { + cmd += " --skip ${UPLIFT_BLACKLIST}" + } + + sh cmd + sh 'git diff' + } +} + +def uplift_review = { + withCredentials([sshUserPrivateKey(credentialsId: 'jenkins-uplifter-key', + keyFileVariable: 'SSH_KEY', + usernameVariable: 'SSH_USER')]) { + + sh 'sudo apt-get install git-review -y' + + sh "cp ${SSH_KEY} ~/.ssh/id_rsa" + + sh "ssh-keyscan -p 29418 review.opendev.org >> ~/.ssh/known_hosts" + + sh "git clone ssh://${SSH_USER}@review.opendev.org:29418/airship/treasuremap" + sh "scp -p -P 29418 ${SSH_USER}@review.opendev.org:hooks/commit-msg treasuremap/.git/hooks/" + + sh "cp global/software/config/versions.yaml treasuremap/global/software/config/versions.yaml" + + dir ('treasuremap') { + sh "git config --global user.name 'Jenkins Uplifter'" + sh "git config --global user.email ${SSH_USER}@gmail.com" + sh "git config --global gitreview.username ${SSH_USER}" + + sh 'git checkout -b versions/uplift' + + sh 'git add global/software/config/versions.yaml' + sh 'git status' + sh 'git commit -m "Auto chart/image uplift to latest"' + + sh 'git review -s' + sh 'git review' + } + } +} + + +//// test utils + +def sanity_tests = { + stage('Sanity Tests') { + + sh 'sudo apt-get install nmap -y' + + withEnv(['TERM_OPTS=-i', + 'OSH_EXT_SUBNET=', + 'OSH_BR_EX_ADDR=', + 'OSH_KEYSTONE_URL=http://keystone.openstack.svc.cluster.local', + 'OSH_EXT_NET_VLAN=75', + 'OSH_EXT_SUBNET=10.22.75.0/24', + 'OSH_EXT_GATEWAY=10.22.75.1', + 'OSH_EXT_SUBNET_POOL_START=10.22.75.11', + 'OSH_EXT_SUBNET_POOL_END=10.22.75.99', + 'OSH_REGION_NAME=airsloop', + 'OSH_ADMIN_PASSWD=airsloop123']) { + sh 'tools/tests.sh' + } + } +} + + +//// main flow + +vm(timeout: 360, + publicNet: 'foundry', + artifactoryLogs: ARTIFACTORY_LOGS.toBoolean()) { + + // wait and make sure genesis is up + ssh.wait (GENESIS_CREDS, GENESIS_IP, 'hostname') + + + // disable docker/kubelet services + // this is done to be able to properly cleanup genesis after reboot + ssh.cmd (GENESIS_CREDS, GENESIS_IP, + 'sudo systemctl disable kubelet') + ssh.cmd (GENESIS_CREDS, GENESIS_IP, + 'sudo systemctl disable docker') + + sh 'sudo apt-get update' + sh 'sudo apt-get install docker.io -y' + + sh 'cat /etc/hosts > hosts' + sh 'echo "10.22.71.21 keystone.ucp.svc.cluster.local ' + + 'shipyard-api.ucp.svc.cluster.local ' + + 'keystone.openstack.svc.cluster.local ' + + 'heat.openstack.svc.cluster.local ' + + 'cinder.openstack.svc.cluster.local ' + + 'cloudformation.openstack.svc.cluster.local ' + + 'glance.openstack.svc.cluster.local ' + + 'glance-reg.openstack.svc.cluster.local ' + + 'horizon.openstack.svc.cluster.local ' + + 'metadata.openstack.svc.cluster.local ' + + 'neutron.openstack.svc.cluster.local ' + + 'nova.openstack.svc.cluster.local ' + + 'placement.openstack.svc.cluster.local" >> hosts' + sh 'sudo mv hosts /etc/hosts' + + reset_bare_metal() + + clone(AIRSHIP_MANIFESTS_REF) + + // use updater tool to pull latest charts/images + if (AIRSHIP_MANIFESTS_REF == 'uplift') { + uplift_versions() + } + + resolve_versions() + + pegleg_site_collect() + prom_config_gen() + + stage ('Genesis Wait') { + ssh.wait (GENESIS_CREDS, GENESIS_IP, 'hostname') + } + + genesis_cleanup() + genesis_deploy() + + timeout(240) { + shipyard_deploy('deploy_site') + } + + sanity_tests() + + if (AIRSHIP_MANIFESTS_REF == 'uplift') { + uplift_review() + } +} + diff --git a/tools/gate/airsloop/seed.groovy b/tools/gate/airsloop/seed.groovy new file mode 100644 index 000000000..6d9dfeba2 --- /dev/null +++ b/tools/gate/airsloop/seed.groovy @@ -0,0 +1,87 @@ + +pipelineJob('airsloop') { + + displayName('Airsloop') + description('Bare-metal minimalistic deployment pipeline') + + logRotator { + daysToKeep(30) + } + + parameters { + string { + defaultValue("uplift") + description("Reference to airship-treasuremap, e.g. refs/changes/12/12345/12") + name("AIRSHIP_MANIFESTS_REF") + trim(true) + } + booleanParam { + defaultValue(true) + description('Flag to publish the console log from the pipeline run to artifactory. ' + + 'Set this value to false, if you should want to suppress uploading ' + + 'and publishing of the pipeline logs to the artifactory.') + name("ARTIFACTORY_LOGS") + } + + } + + + concurrentBuild(false) + + triggers { + gerritTrigger { + serverName('OS-CommunityGerrit') + silentMode(true) + + gerritProjects { + gerritProject { + compareType('PLAIN') + pattern("airship/treasuremap") + branches { + branch { + compareType("ANT") + pattern("**") + } + } + disableStrictForbiddenFileVerification(false) + + filePaths { + filePath { + compareType('ANT') + pattern('global/**') + } + filePath { + compareType('ANT') + pattern('type/sloop/**') + } + filePath { + compareType('ANT') + pattern('site/airsloop/**') + } + } + } + } + + triggerOnEvents { + patchsetCreated { + excludeDrafts(false) + excludeTrivialRebase(false) + excludeNoCodeChange(false) + } + commentAddedContains { + commentAddedCommentContains('recheck') + } + } + + cron('H H * * *') + } + + definition { + cps { + script(readFileFromWorkspace("tools/gate/airsloop/Jenkinsfile")) + sandbox(false) + } + } + } +} +