You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
360 lines
9.7 KiB
360 lines
9.7 KiB
// Pipeline expects genesis to be setup according to the documentation |
|
// including networks, disks, ntp, ip rules, etc.. manually. |
|
// https://airship-treasuremap.readthedocs.io/en/latest/authoring_and_deployment.html#genesis-node |
|
|
|
// shared libaries used within the pipeline |
|
// https://github.com/att-comdev/cicd/blob/master/vars |
|
|
|
import org.yaml.snakeyaml.Yaml |
|
import org.yaml.snakeyaml.DumperOptions; |
|
|
|
import groovy.json.JsonSlurperClassic |
|
import groovy.json.JsonOutput |
|
|
|
PEGLEG_IMAGE = 'quay.io/airshipit/pegleg:178c058474fb632806e281673d3eaf6be80fa854' |
|
|
|
COLLECT_DIR = 'seaworthy' |
|
BUNDLE_DIR = 'bundle' |
|
|
|
KEYSTONE_URL = 'https://iam-sw.atlantafoundry.com' |
|
SHIPYARD_URL = 'https://shipyard-sw.atlantafoundry.com/api/v1.0' |
|
SHIPYARD_CREDS = 'seaworthy-shipyard-iam-pw' |
|
|
|
SITE_NAME='seaworthy' |
|
IPMI_CREDS = 'seaworthy-ipmi' |
|
|
|
GENESIS_IP = '10.23.21.11' |
|
GENESIS_CREDS = 'seaworthy-key' |
|
|
|
GENESIS_IPMI_IP = '10.23.104.11' |
|
GENESIS_CEPH_DISKS = ['b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k'] |
|
|
|
IPMI_IPS = ['10.23.104.12', |
|
'10.23.104.13', |
|
'10.23.104.14', |
|
'10.23.104.17', |
|
'10.23.104.19'] |
|
|
|
AIRSHIP_MANIFESTS_REPO = 'https://review.opendev.org/airship/treasuremap' |
|
|
|
if (env.GERRIT_REFSPEC) { |
|
AIRSHIP_MANIFESTS_REF = GERRIT_REFSPEC |
|
DISPLAY_NAME = GERRIT_EVENT_TYPE |
|
} else if (AIRSHIP_MANIFESTS_REF == 'uplift') { |
|
DISPLAY_NAME = 'uplift' |
|
} else { |
|
DISPLAY_NAME = "manual ${AIRSHIP_MANIFESTS_REF}" |
|
} |
|
|
|
currentBuild.displayName = "#${BUILD_NUMBER} ${DISPLAY_NAME}" |
|
|
|
|
|
//// git utils |
|
|
|
def clone(String ref){ |
|
|
|
def refspec = '' |
|
|
|
// override refspec if patchset provided |
|
if (ref.contains('refs')) { |
|
refspec = "${ref}:${ref}" |
|
} |
|
|
|
// base uplift on latest master |
|
if (ref == 'uplift') { |
|
ref = 'master' |
|
} |
|
|
|
checkout poll: false, |
|
scm: [$class: 'GitSCM', |
|
branches: [[name: ref]], |
|
doGenerateSubmoduleConfigurations: false, |
|
extensions: [[$class: 'CleanBeforeCheckout']], |
|
submoduleCfg: [], |
|
userRemoteConfigs: [[refspec: refspec, |
|
url: AIRSHIP_MANIFESTS_REPO ]]] |
|
} |
|
|
|
|
|
//// bare-metal utils |
|
|
|
def reset_bare_metal = { |
|
stage ('Bare-Metal Reset') { |
|
|
|
withCredentials([usernamePassword(credentialsId: IPMI_CREDS, |
|
usernameVariable: 'IUSER', |
|
passwordVariable: 'IPASS')]) { |
|
|
|
def auth = redfish.getBasicAuth(IUSER, IPASS) |
|
|
|
// power cycle genesis |
|
redfish.powerOff(GENESIS_IPMI_IP, auth) |
|
redfish.powerOn(GENESIS_IPMI_IP, auth) |
|
|
|
// shutdown all other nodes |
|
IPMI_IPS.each() { |
|
redfish.powerOff(it, auth) |
|
} |
|
} |
|
|
|
} |
|
} |
|
|
|
|
|
//// manifest utils |
|
|
|
def resolve_versions = { |
|
def shipyard = readYaml file: "site/${SITE_NAME}/secrets/passphrases/ucp_shipyard_keystone_password.yaml" |
|
SHIPYARD_PASSWD = shipyard.data |
|
} |
|
|
|
def pegleg_site_collect = { |
|
stage('Pegleg Site Collect') { |
|
|
|
configFileProvider([configFile(fileId: 'seaworthy-site-definition', |
|
targetLocation: 'site/seaworthy/site-definition.yaml')]) { |
|
} |
|
|
|
withCredentials([sshUserPrivateKey(credentialsId: 'jenkins-attcomdev-key', |
|
keyFileVariable: 'SSH_KEY', |
|
usernameVariable: 'SSH_USER')]) { |
|
|
|
sh "cp ${SSH_KEY} ssh-key" |
|
|
|
auth = "-u ${SSH_USER} -k /target/ssh-key" |
|
cmd = "pegleg site ${auth} -r /target collect ${SITE_NAME} -s /target/${COLLECT_DIR}" |
|
sh "sudo docker run --rm -t -v \$(pwd):/target ${PEGLEG_IMAGE} ${cmd}" |
|
} |
|
|
|
sh "tar czf ${COLLECT_DIR}.tar.gz ${COLLECT_DIR}" |
|
archiveArtifacts "${COLLECT_DIR}.tar.gz" |
|
} |
|
} |
|
|
|
def prom_config_gen = { |
|
stage ("Promenade Config Gen") { |
|
|
|
withEnv(['TERM_OPTS=-t']) { |
|
sh "mkdir -p ${BUNDLE_DIR}" |
|
sh "sudo -E tools/airship promenade build-all --validators" + |
|
" -o ${BUNDLE_DIR} /target/${COLLECT_DIR}/*.yaml" |
|
} |
|
|
|
sh "tar czf ${BUNDLE_DIR}.tar.gz ${BUNDLE_DIR}" |
|
archiveArtifacts "${BUNDLE_DIR}.tar.gz" |
|
} |
|
} |
|
|
|
|
|
//// genesis utils |
|
|
|
def genesis_cleanup = { |
|
stage('Genesis Cleanup') { |
|
|
|
dfiles = ['promenade', |
|
BUNDLE_DIR, |
|
"${BUNDLE_DIR}.tar.gz", |
|
SITE_NAME, |
|
'seaworthy.tar.gz', |
|
'debug-cab23-r720-11.tgz', |
|
'/var/lib/docker', |
|
'/var/lib/kubelet'] |
|
|
|
dfiles.each() { |
|
ssh.cmd (GENESIS_CREDS, GENESIS_IP, "sudo rm -rf ${it}") |
|
} |
|
|
|
ssh.cmd (GENESIS_CREDS, GENESIS_IP, |
|
'git clone https://opendev.org/airship/promenade') |
|
ssh.cmd (GENESIS_CREDS, GENESIS_IP, |
|
'sudo -S promenade/tools/cleanup.sh -f') |
|
|
|
GENESIS_CEPH_DISKS.each() { |
|
ssh.cmd(GENESIS_CREDS, GENESIS_IP, |
|
"sudo parted -s /dev/sd${it} mklabel gpt") |
|
} |
|
} |
|
} |
|
|
|
def debug_report = { |
|
ssh.cmd (GENESIS_CREDS, GENESIS_IP, 'sudo debug-report.sh') |
|
ssh.get (GENESIS_CREDS, GENESIS_IP, 'debug-cab23-r720-11.tgz', '.') |
|
|
|
archiveArtifacts 'debug-cab23-r720-11.tgz' |
|
} |
|
|
|
def genesis_deploy = { |
|
stage('Genesis Deploy') { |
|
|
|
try { |
|
ssh.put(GENESIS_CREDS, GENESIS_IP, "${BUNDLE_DIR}.tar.gz", '.') |
|
ssh.cmd(GENESIS_CREDS, GENESIS_IP, "tar xvzf ${BUNDLE_DIR}.tar.gz") |
|
|
|
timeout (90) { |
|
ssh.cmd(GENESIS_CREDS, GENESIS_IP, |
|
"sudo ${BUNDLE_DIR}/genesis.sh") |
|
|
|
// fixme: there is notable initial slowness likely due to coredns |
|
// going out of service and taking time to recover |
|
// this is a long time issue and needs to be taken look at |
|
retry(2) { |
|
ssh.cmd(GENESIS_CREDS, GENESIS_IP, |
|
"sudo -S ${BUNDLE_DIR}/validate-genesis.sh") |
|
} |
|
} |
|
|
|
sleep 120 // wait for k8s to calm down |
|
|
|
} catch (err) { |
|
debug_report() |
|
error(err) |
|
} |
|
} |
|
} |
|
|
|
def shipyard_deploy = { action -> |
|
try { |
|
uuid = UUID.randomUUID().toString() |
|
def req = keystone.retrieveToken(SHIPYARD_CREDS, KEYSTONE_URL) |
|
def token = req.getHeaders()["X-Subject-Token"][0] |
|
shipyard2.uploadConfig(uuid, token, SHIPYARD_URL, SITE_NAME) |
|
|
|
shipyard2.waitAction(action: action, |
|
uuid: uuid, |
|
shipyardUrl: SHIPYARD_URL, |
|
keystoneCreds: SHIPYARD_CREDS, |
|
keystoneUrl: KEYSTONE_URL) |
|
|
|
} catch (err) { |
|
print err |
|
debug_report() |
|
throw err |
|
} |
|
} |
|
|
|
|
|
//// uplift utils |
|
|
|
def uplift_versions = { |
|
stage('Uplift Versions') { |
|
sh 'sudo apt-get install python3-yaml python3-git -y' |
|
|
|
def cmd = 'tools/updater.py --in-file global/software/config/versions.yaml --tag-filter ubuntu' |
|
|
|
if (!UPLIFT_BLACKLIST.isEmpty()) { |
|
cmd += " --skip ${UPLIFT_BLACKLIST}" |
|
} |
|
|
|
sh cmd |
|
sh 'git diff' |
|
} |
|
} |
|
|
|
def uplift_review = { |
|
withCredentials([sshUserPrivateKey(credentialsId: 'jenkins-uplifter-key', |
|
keyFileVariable: 'SSH_KEY', |
|
usernameVariable: 'SSH_USER')]) { |
|
|
|
sh 'sudo apt-get install git-review -y' |
|
|
|
sh "cp ${SSH_KEY} ~/.ssh/id_rsa" |
|
|
|
sh "ssh-keyscan -p 29418 review.opendev.org >> ~/.ssh/known_hosts" |
|
|
|
sh "git clone ssh://${SSH_USER}@review.opendev.org:29418/airship/treasuremap" |
|
sh "scp -p -P 29418 ${SSH_USER}@review.opendev.org:hooks/commit-msg treasuremap/.git/hooks/" |
|
|
|
sh "cp global/software/config/versions.yaml treasuremap/global/software/config/versions.yaml" |
|
|
|
dir ('treasuremap') { |
|
sh "git config --global user.name 'Jenkins Uplifter'" |
|
sh "git config --global user.email ${SSH_USER}@gmail.com" |
|
sh "git config --global gitreview.username ${SSH_USER}" |
|
|
|
sh 'git checkout -b versions/uplift' |
|
|
|
sh 'git add global/software/config/versions.yaml' |
|
sh 'git status' |
|
sh 'git commit -m "Auto chart/image uplift to latest"' |
|
|
|
sh 'git review -s' |
|
sh 'git review' |
|
} |
|
} |
|
} |
|
|
|
|
|
//// test utils |
|
|
|
def sanity_tests = { |
|
stage('Sanity Tests') { |
|
|
|
sh 'sudo apt-get install nmap -y' |
|
|
|
withEnv(['TERM_OPTS=-i', |
|
'OSH_EXT_SUBNET=', |
|
'OSH_BR_EX_ADDR=', |
|
'OSH_KEYSTONE_URL=https://identity-sw.atlantafoundry.com/v3', |
|
'OSH_REGION_NAME=seaworthy', |
|
'OSH_ADMIN_PASSWD=password123']) { |
|
sh 'tools/tests.sh' |
|
} |
|
} |
|
} |
|
|
|
|
|
//// main flow |
|
|
|
vm(image: 'ubuntu-16.04-server-cloudimg-amd64', |
|
timeout: 360, |
|
publicNet: 'foundry', |
|
buildType: 'flat', |
|
initScript: 'cloud-config', |
|
artifactoryLogs: ARTIFACTORY_LOGS.toBoolean()) { |
|
|
|
// wait and make sure genesis is up |
|
ssh.wait (GENESIS_CREDS, GENESIS_IP, 'hostname') |
|
|
|
|
|
// disable docker/kubelet services |
|
// this is done to be able to properly cleanup genesis after reboot |
|
ssh.cmd (GENESIS_CREDS, GENESIS_IP, |
|
'sudo systemctl disable kubelet') |
|
ssh.cmd (GENESIS_CREDS, GENESIS_IP, |
|
'sudo systemctl disable docker') |
|
|
|
sh 'sudo apt-get update' |
|
sh 'sudo apt-get install docker.io -y' |
|
|
|
reset_bare_metal() |
|
|
|
clone(AIRSHIP_MANIFESTS_REF) |
|
|
|
// use updater tool to pull latest charts/images |
|
if (AIRSHIP_MANIFESTS_REF == 'uplift') { |
|
uplift_versions() |
|
} |
|
|
|
resolve_versions() |
|
|
|
pegleg_site_collect() |
|
prom_config_gen() |
|
|
|
stage ('Genesis Wait') { |
|
ssh.wait (GENESIS_CREDS, GENESIS_IP, 'hostname') |
|
} |
|
|
|
genesis_cleanup() |
|
genesis_deploy() |
|
|
|
timeout(240) { |
|
shipyard_deploy('deploy_site') |
|
} |
|
|
|
sanity_tests() |
|
|
|
if (AIRSHIP_MANIFESTS_REF == 'uplift') { |
|
uplift_review() |
|
} |
|
} |
|
|
|
|