Initial draft for airship-seaworthy pipeline
Change-Id: Ifdd3b1e18eaa03809d311d31357a6d629e03f786
This commit is contained in:
parent
5c20bfa377
commit
b9796bd9a1
|
@ -0,0 +1,415 @@
|
|||
|
||||
// Pipeline expects genesis to be setup according to the documentation
|
||||
// including network, ntp, ip rules, etc.
|
||||
// http://treasuremap.readthedocs.io/en/latest/deployment.html#genesis-node
|
||||
|
||||
import org.yaml.snakeyaml.Yaml
|
||||
import org.yaml.snakeyaml.DumperOptions;
|
||||
|
||||
import groovy.json.JsonSlurperClassic
|
||||
import groovy.json.JsonOutput
|
||||
|
||||
PEGLEG_IMAGE='quay.io/airshipit/pegleg:54e30687d444e83cbc628a2dfb4bb3cb0815328f'
|
||||
|
||||
KEYSTONE_URL = 'http://keystone.ucp.svc.cluster.local:80/v3/auth/tokens'
|
||||
SHIPYARD_URL = 'http://shipyard-api.ucp.svc.cluster.local/api/v1.0'
|
||||
|
||||
ARTF_PATH="cicd/${JOB_BASE_NAME}/${BUILD_NUMBER}"
|
||||
|
||||
SITE_NAME='airship-seaworthy'
|
||||
SHIPYARD_CREDS = 'airship-seaworthy-shipyard-iam-pw'
|
||||
IPMI_CREDS = 'airship-seaworthy-ipmi'
|
||||
GENESIS_NODE = 'airship-seaworthy-genesis'
|
||||
|
||||
GENESIS_IPMI_IP = '10.23.104.11'
|
||||
GENESIS_COREDNS_IP = '10.23.22.11'
|
||||
GENESIS_CEPH_DISKS = ['c', 'd', 'e', 'f', 'g', 'h']
|
||||
|
||||
|
||||
IPMI_IPS = ['10.23.104.12',
|
||||
'10.23.104.13',
|
||||
'10.23.104.14',
|
||||
'10.23.104.17',
|
||||
'10.23.104.19']
|
||||
|
||||
if (env.GERRIT_REFSPEC) {
|
||||
AIRSHIP_MANIFESTS = GERRIT_REFSPEC
|
||||
}
|
||||
|
||||
|
||||
//// container utils
|
||||
|
||||
def pod_config = {
|
||||
return """
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
spec:
|
||||
securityContext:
|
||||
runAsUser: 10000
|
||||
"""
|
||||
}
|
||||
|
||||
|
||||
//// artf utils
|
||||
|
||||
def artf = Artifactory.server 'artifactory'
|
||||
|
||||
def artf_spec = { pattern, target ->
|
||||
spec = ['files': [['pattern': pattern,
|
||||
'target': target,
|
||||
'flat': true]]]
|
||||
return new JsonOutput().toJson(spec)
|
||||
}
|
||||
|
||||
def artf_publish = { pattern, target ->
|
||||
info = artf.upload(artf_spec(pattern, target))
|
||||
artf.publishBuildInfo(info)
|
||||
}
|
||||
|
||||
def artf_download = { pattern, target ->
|
||||
artf.download(artf_spec(pattern, target))
|
||||
}
|
||||
|
||||
|
||||
//// manifest utils
|
||||
|
||||
def pegleg_collect = {
|
||||
stage ('Pegleg Collect') {
|
||||
sh 'mkdir -p manifests bundle'
|
||||
sh 'pegleg site -p airship-seaworthy collect airship-seaworthy -s output'
|
||||
|
||||
sh 'tar czf manifests.tar.gz manifests'
|
||||
artf_publish('manifests.tar.gz', "${ARTF_PATH}/")
|
||||
}
|
||||
}
|
||||
|
||||
def prom_gen = {
|
||||
stage ("Promenade Certs Gen") {
|
||||
// note: certs are generated for verification purpose only
|
||||
// deployment will use certs merged into security repo
|
||||
sh 'mkdir certs'
|
||||
sh 'promenade generate-certs -o certs/ manifests/*.yaml'
|
||||
}
|
||||
stage ("Promenade Bundle Gen") {
|
||||
sh "promenade build-all --validators -o bundle manifests/*.yaml"
|
||||
|
||||
sh 'tar czf bundle.tar.gz bundle'
|
||||
artf_publish('bundle.tar.gz', "${ARTF_PATH}/")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//// prepare manifests
|
||||
|
||||
def process_manifests = {
|
||||
vm2() {
|
||||
clone('openstack/airship-treasuremap', AIRSHIP_MANIFESTS)
|
||||
|
||||
stage ('Bare-Metal Reset') {
|
||||
|
||||
sh 'apt-get update'
|
||||
sh 'apt-get install -y ipmitool'
|
||||
|
||||
withCredentials([usernamePassword(credentialsId: IPMI_CREDS,
|
||||
usernameVariable: 'IUSER',
|
||||
passwordVariable: 'IPASS')]) {
|
||||
IPMI_IPS.each() {
|
||||
withCredentials([usernamePassword(credentialsId: IPMI_CREDS,
|
||||
usernameVariable: 'IUSER',
|
||||
passwordVariable: 'IPASS')]) {
|
||||
sh "ipmitool -I lanplus -H ${it} -U \$IUSER -P \$IPASS chassis power off"
|
||||
}
|
||||
}
|
||||
|
||||
sh "ipmitool -I lanplus -H ${GENESIS_IPMI_IP} -U \$IUSER -P \$IPASS chassis power off"
|
||||
sh "ipmitool -I lanplus -H ${GENESIS_IPMI_IP} -U \$IUSER -P \$IPASS chassis power on"
|
||||
}
|
||||
}
|
||||
|
||||
stage ('Resolve Versions') {
|
||||
|
||||
def versions = readYaml file: 'global/v4.0/software/config/versions.yaml'
|
||||
PROMENADE_IMAGE = versions.data.images.ucp.promenade.promenade
|
||||
|
||||
def shipyard = readYaml file: "site/${SITE_NAME}/secrets/passphrases/ucp_shipyard_keystone_password.yaml"
|
||||
SHIPYARD_PASSWD = shipyard.data
|
||||
|
||||
def ipmi = readYaml file: "site/${SITE_NAME}/secrets/passphrases/ipmi_admin_password.yaml"
|
||||
IPMI_PASSWD = ipmi.data
|
||||
}
|
||||
|
||||
pegleg_collect()
|
||||
prom_gen()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//// genesis utils
|
||||
|
||||
def genesis_wait = {
|
||||
stage('Genesis Wait') {
|
||||
timeout(12) {
|
||||
node(GENESIS_NODE) {
|
||||
sh 'hostname'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def genesis_cleanup = {
|
||||
stage('Genesis Cleanup') {
|
||||
sh "sudo -S airship-seaworthy/tools/cleanup.sh -f"
|
||||
|
||||
GENESIS_CEPH_DISKS.each() {
|
||||
sh "sudo parted -s /dev/sd${it} mklabel msdos"
|
||||
}
|
||||
sh 'sudo rm -rf /var/lib/ceph/journal/osd'
|
||||
|
||||
// reset hosts dns servers
|
||||
sh "sudo sed -i '/${GENESIS_COREDNS_IP}/d' /etc/resolv.conf"
|
||||
}
|
||||
}
|
||||
|
||||
def genesis_run = {
|
||||
stage('Genesis Deploy') {
|
||||
artf_download("${ARTF_PATH}/bundle.tar.gz", "")
|
||||
sh 'tar xzf bundle.tar.gz'
|
||||
|
||||
dir ('bundle') {
|
||||
timeout (90) {
|
||||
sh 'sudo ./genesis.sh'
|
||||
sh 'sudo -S ./validate-genesis.sh'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sh 'sudo kubectl get pods --all-namespaces -o wide'
|
||||
}
|
||||
|
||||
def debug_report = {
|
||||
node(GENESIS_NODE) {
|
||||
sh "sudo debug-report.sh"
|
||||
artf_publish("debug-cab23-r720-11.tgz", "${ARTF_PATH}/")
|
||||
}
|
||||
}
|
||||
|
||||
def genesis_deploy = {
|
||||
node(GENESIS_NODE) {
|
||||
|
||||
genesis_cleanup()
|
||||
|
||||
try {
|
||||
genesis_run()
|
||||
sleep 47 // wait for k8s to calm down
|
||||
|
||||
} catch (err) {
|
||||
print err
|
||||
debug_report()
|
||||
error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//// keystone utils
|
||||
|
||||
def keystone_token = {
|
||||
|
||||
withCredentials([[$class: 'UsernamePasswordMultiBinding',
|
||||
credentialsId: SHIPYARD_CREDS,
|
||||
usernameVariable: 'user',
|
||||
passwordVariable: 'pass']]) {
|
||||
|
||||
def req = ['auth': [
|
||||
'identity': [
|
||||
'methods': ['password'],
|
||||
'password': [
|
||||
'user': ['name': user,
|
||||
'domain': ['id': 'default'],
|
||||
'password': pass]]]]]
|
||||
|
||||
def jreq = new JsonOutput().toJson(req)
|
||||
|
||||
def res = httpRequest (url: KEYSTONE_URL,
|
||||
contentType: 'APPLICATION_JSON',
|
||||
httpMode: 'POST',
|
||||
requestBody: jreq)
|
||||
|
||||
return res.getHeaders()['X-Subject-Token'][0]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//// shipyard utils
|
||||
|
||||
def shipyard_config_upload = {
|
||||
|
||||
def token = keystone_token()
|
||||
|
||||
stage('Shipyard Config Create') {
|
||||
|
||||
artf_download("${ARTF_PATH}/manifests.tar.gz", "")
|
||||
sh "tar xzf manifests.tar.gz"
|
||||
|
||||
def manifests = readFile 'manifests/airship-manifests.yaml'
|
||||
|
||||
def res = httpRequest (url: "${SHIPYARD_URL}/configdocs/${SITE_NAME}?buffermode=replace",
|
||||
httpMode: 'POST',
|
||||
customHeaders: [[name: 'Content-Type', value: 'application/x-yaml'],
|
||||
[name: 'X-Auth-Token', value: token]],
|
||||
requestBody: manifests)
|
||||
|
||||
if (res.status != 201) {
|
||||
error("Failed to upload configdocs: ${res.status}")
|
||||
}
|
||||
print res.content
|
||||
}
|
||||
|
||||
stage('Shipyard Config Commit') {
|
||||
def res = httpRequest (url: "${SHIPYARD_URL}/commitconfigdocs", httpMode: 'POST',
|
||||
customHeaders: [[name: 'X-Auth-Token', value: token]])
|
||||
if (res.status != 200) {
|
||||
error("Failed to commit configdocs: ${res.status}")
|
||||
}
|
||||
print res.content
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def shipyard_steps_get = { action ->
|
||||
|
||||
def res = httpRequest (url: "${SHIPYARD_URL}/actions/${action}",
|
||||
contentType: 'APPLICATION_JSON',
|
||||
httpMode: 'GET',
|
||||
customHeaders: [[name: 'X-Auth-Token', value: keystone_token()]])
|
||||
|
||||
if (res.status != 200) {
|
||||
error("Failed to get Shypyard action steps: ${res.status}")
|
||||
}
|
||||
|
||||
def cont = new JsonSlurperClassic().parseText(res.content)
|
||||
print cont
|
||||
|
||||
return cont.steps
|
||||
}
|
||||
|
||||
|
||||
def shipyard_step_wait = { systep, interval ->
|
||||
|
||||
print ">> Waiting on Shipyard step: ${systep}"
|
||||
|
||||
def String state = systep.state
|
||||
def errcount = 0
|
||||
def res
|
||||
|
||||
while (state == null || state == 'running' || state == 'queued' || state == 'scheduled') {
|
||||
sleep interval
|
||||
|
||||
if (errcount > 3) {
|
||||
print "Multiple re-tries done already - giving up!"
|
||||
break
|
||||
}
|
||||
|
||||
res = httpRequest (url: "${SHIPYARD_URL}${systep.url}",
|
||||
contentType: 'APPLICATION_JSON',
|
||||
httpMode: 'GET',
|
||||
customHeaders: [[name: 'X-Auth-Token', value: keystone_token()]])
|
||||
|
||||
if (!res) {
|
||||
print "httpRequest returned null - likely library issue"
|
||||
errcount+=1
|
||||
continue
|
||||
}
|
||||
|
||||
if (res.status != 200) {
|
||||
print "Failed to get Shipyard step info: ${res.status}"
|
||||
errcount+=1
|
||||
continue
|
||||
}
|
||||
|
||||
if (!res.content) {
|
||||
print "Shypyard returned null content"
|
||||
errcount+=1
|
||||
continue
|
||||
}
|
||||
|
||||
def cont = new JsonSlurperClassic().parseText(res.content)
|
||||
print cont
|
||||
state = cont.state
|
||||
}
|
||||
|
||||
if (state != 'success' && state != 'skipped') {
|
||||
error("Failed Shipyard task ${systep.id}: ${res.status}, ${res.content}")
|
||||
}
|
||||
}
|
||||
|
||||
def shipyard_steps_wait = { action ->
|
||||
|
||||
def systeps = shipyard_steps_get(action)
|
||||
|
||||
systeps.each() {
|
||||
// add explicit stage for steps taking long time
|
||||
if (it.id == 'drydock_build' || it.id == 'armada_build') {
|
||||
stage ("Shipyard (${it.id})") {
|
||||
shipyard_step_wait(it, 240)
|
||||
}
|
||||
} else {
|
||||
shipyard_step_wait(it, 4)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def shipyard_action_create = { action ->
|
||||
|
||||
def req = [ 'name': action ]
|
||||
def jreq = new JsonOutput().toJson(req)
|
||||
|
||||
res = httpRequest (url: "${SHIPYARD_URL}/actions",
|
||||
contentType: 'APPLICATION_JSON',
|
||||
httpMode: 'POST',
|
||||
customHeaders: [[name: 'X-Auth-Token', value: keystone_token()]],
|
||||
requestBody: jreq)
|
||||
|
||||
if (res.status != 201) {
|
||||
error("Failed to create Shypyard action: ${res.status}")
|
||||
}
|
||||
|
||||
def cont = new JsonSlurperClassic().parseText(res.content)
|
||||
print cont
|
||||
|
||||
shipyard_steps_wait(cont.id)
|
||||
}
|
||||
|
||||
def shipyard_deploy = { action ->
|
||||
node(GENESIS_NODE) {
|
||||
try {
|
||||
shipyard_config_upload()
|
||||
shipyard_action_create(action)
|
||||
|
||||
} catch (err) {
|
||||
print err
|
||||
debug_report()
|
||||
error(err)
|
||||
}
|
||||
|
||||
// debug_report()
|
||||
sh 'sudo kubectl get pods --all-namespaces -o wide'
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//// execute workflow
|
||||
|
||||
bare_metal_reset()
|
||||
resolve_versions()
|
||||
process_manifests()
|
||||
|
||||
genesis_wait()
|
||||
genesis_deploy()
|
||||
|
||||
shipyard_deploy('deploy_site')
|
||||
|
||||
// verify update on unchanged docs
|
||||
// tests general shipyard workflow on site update
|
||||
// shipyard_deploy('update_site')
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
|
||||
pipelineJob('airship-seaworthy') {
|
||||
|
||||
displayName('Airship Seaworthy - bare-metal continuous deployment pipeline')
|
||||
description('')
|
||||
|
||||
parameters {
|
||||
string {
|
||||
defaultValue("master")
|
||||
description("Reference to airship-treasuremap, e.g. refs/changes/12/12345/12")
|
||||
name("AIRSHIP_MANIFESTS")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
concurrentBuild(false)
|
||||
|
||||
triggers {
|
||||
gerritTrigger {
|
||||
serverName('fixme-gerrit')
|
||||
gerritProjects {
|
||||
gerritProject {
|
||||
compareType('PLAIN')
|
||||
pattern("openstack/airship-treasuremap")
|
||||
branches {
|
||||
branch {
|
||||
compareType("ANT")
|
||||
pattern("**")
|
||||
}
|
||||
}
|
||||
disableStrictForbiddenFileVerification(false)
|
||||
}
|
||||
}
|
||||
triggerOnEvents {
|
||||
patchsetCreated {
|
||||
excludeDrafts(false)
|
||||
excludeTrivialRebase(false)
|
||||
excludeNoCodeChange(false)
|
||||
}
|
||||
commentAddedContains {
|
||||
commentAddedCommentContains('recheck')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
definition {
|
||||
cps {
|
||||
script(readFileFromWorkspace("tools/gate/Jenkinsfile"))
|
||||
sandbox()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue