Reference Airship manifests, CICD, and reference architecture.
Vous ne pouvez pas sélectionner plus de 25 sujets Les noms de sujets doivent commencer par une lettre ou un nombre, peuvent contenir des tirets ('-') et peuvent comporter jusqu'à 35 caractères.

358 lignes
9.6KB

  1. // Pipeline expects genesis to be setup according to the documentation
  2. // including networks, disks, ntp, ip rules, etc.. manually.
  3. // https://airship-treasuremap.readthedocs.io/en/latest/authoring_and_deployment.html#genesis-node
  4. // shared libaries used within the pipeline
  5. // https://github.com/att-comdev/cicd/blob/master/vars
  6. import org.yaml.snakeyaml.Yaml
  7. import org.yaml.snakeyaml.DumperOptions;
  8. import groovy.json.JsonSlurperClassic
  9. import groovy.json.JsonOutput
  10. COLLECT_DIR = 'seaworthy'
  11. BUNDLE_DIR = 'bundle'
  12. KEYSTONE_URL = 'https://iam-sw.atlantafoundry.com'
  13. SHIPYARD_URL = 'https://shipyard-sw.atlantafoundry.com/api/v1.0'
  14. SHIPYARD_CREDS = 'seaworthy-shipyard-iam-pw'
  15. SITE_NAME='seaworthy'
  16. IPMI_CREDS = 'seaworthy-ipmi'
  17. GENESIS_IP = '10.23.21.11'
  18. GENESIS_CREDS = 'seaworthy-key'
  19. GENESIS_IPMI_IP = '10.23.104.11'
  20. GENESIS_CEPH_DISKS = ['b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k']
  21. IPMI_IPS = ['10.23.104.12',
  22. '10.23.104.13',
  23. '10.23.104.14',
  24. '10.23.104.16',
  25. '10.23.104.17']
  26. AIRSHIP_MANIFESTS_REPO = 'https://review.opendev.org/airship/treasuremap'
  27. if (env.GERRIT_REFSPEC) {
  28. AIRSHIP_MANIFESTS_REF = GERRIT_REFSPEC
  29. DISPLAY_NAME = GERRIT_EVENT_TYPE
  30. } else if (AIRSHIP_MANIFESTS_REF == 'uplift') {
  31. DISPLAY_NAME = 'uplift'
  32. } else {
  33. DISPLAY_NAME = "manual ${AIRSHIP_MANIFESTS_REF}"
  34. }
  35. currentBuild.displayName = "#${BUILD_NUMBER} ${DISPLAY_NAME}"
  36. //// git utils
  37. def clone(String ref){
  38. def refspec = ''
  39. // override refspec if patchset provided
  40. if (ref.contains('refs')) {
  41. refspec = "${ref}:${ref}"
  42. }
  43. // base uplift on latest master
  44. if (ref == 'uplift') {
  45. ref = 'master'
  46. }
  47. checkout poll: false,
  48. scm: [$class: 'GitSCM',
  49. branches: [[name: ref]],
  50. doGenerateSubmoduleConfigurations: false,
  51. extensions: [[$class: 'CleanBeforeCheckout']],
  52. submoduleCfg: [],
  53. userRemoteConfigs: [[refspec: refspec,
  54. url: AIRSHIP_MANIFESTS_REPO ]]]
  55. }
  56. //// bare-metal utils
  57. def reset_bare_metal = {
  58. stage ('Bare-Metal Reset') {
  59. withCredentials([usernamePassword(credentialsId: IPMI_CREDS,
  60. usernameVariable: 'IUSER',
  61. passwordVariable: 'IPASS')]) {
  62. def auth = redfish.getBasicAuth(IUSER, IPASS)
  63. // power cycle genesis
  64. redfish.powerOff(GENESIS_IPMI_IP, auth)
  65. redfish.powerOn(GENESIS_IPMI_IP, auth)
  66. // shutdown all other nodes
  67. IPMI_IPS.each() {
  68. redfish.powerOff(it, auth)
  69. }
  70. }
  71. }
  72. }
  73. //// manifest utils
  74. def pegleg_site_collect = {
  75. stage('Pegleg Site Collect') {
  76. configFileProvider([configFile(fileId: 'seaworthy-site-definition',
  77. targetLocation: 'site/seaworthy/site-definition.yaml')]) {
  78. }
  79. withCredentials([sshUserPrivateKey(credentialsId: 'jenkins-attcomdev-key',
  80. keyFileVariable: 'SSH_KEY',
  81. usernameVariable: 'SSH_USER')]) {
  82. sh "cp ${SSH_KEY} ssh-key"
  83. sh "sudo -E tools/airship pegleg site" +
  84. " -u ${SSH_USER}" +
  85. " -k /target/ssh-key" +
  86. " -r /target collect ${SITE_NAME}" +
  87. " -s /target/${COLLECT_DIR}"
  88. sh "sudo chown -R ubuntu:ubuntu ${COLLECT_DIR}"
  89. }
  90. sh "tar czf ${COLLECT_DIR}.tar.gz ${COLLECT_DIR}"
  91. archiveArtifacts "${COLLECT_DIR}.tar.gz"
  92. }
  93. }
  94. def prom_config_gen = {
  95. stage ("Promenade Config Gen") {
  96. withEnv(['TERM_OPTS=-t']) {
  97. sh "mkdir -p ${BUNDLE_DIR}"
  98. sh "sudo -E tools/airship promenade build-all --validators" +
  99. " -o ${BUNDLE_DIR} /target/${COLLECT_DIR}/*.yaml"
  100. }
  101. sh "tar czf ${BUNDLE_DIR}.tar.gz ${BUNDLE_DIR}"
  102. archiveArtifacts "${BUNDLE_DIR}.tar.gz"
  103. }
  104. }
  105. //// genesis utils
  106. def genesis_cleanup = {
  107. stage('Genesis Cleanup') {
  108. dfiles = ['promenade',
  109. BUNDLE_DIR,
  110. "${BUNDLE_DIR}.tar.gz",
  111. SITE_NAME,
  112. 'seaworthy.tar.gz',
  113. 'debug-cab23-r720-11.tgz',
  114. '/var/lib/docker',
  115. '/var/lib/kubelet']
  116. dfiles.each() {
  117. ssh.cmd (GENESIS_CREDS, GENESIS_IP, "sudo rm -rf ${it}")
  118. }
  119. ssh.cmd (GENESIS_CREDS, GENESIS_IP,
  120. 'git clone https://opendev.org/airship/promenade')
  121. ssh.cmd (GENESIS_CREDS, GENESIS_IP,
  122. 'sudo -S promenade/tools/cleanup.sh -f')
  123. GENESIS_CEPH_DISKS.each() {
  124. ssh.cmd(GENESIS_CREDS, GENESIS_IP,
  125. "sudo parted -s /dev/sd${it} mklabel gpt")
  126. }
  127. }
  128. }
  129. def debug_report = {
  130. ssh.cmd (GENESIS_CREDS, GENESIS_IP, 'sudo debug-report.sh')
  131. ssh.get (GENESIS_CREDS, GENESIS_IP, 'debug-cab23-r720-11.tgz', '.')
  132. archiveArtifacts 'debug-cab23-r720-11.tgz'
  133. }
  134. def genesis_deploy = {
  135. stage('Genesis Deploy') {
  136. try {
  137. ssh.put(GENESIS_CREDS, GENESIS_IP, "${BUNDLE_DIR}.tar.gz", '.')
  138. ssh.cmd(GENESIS_CREDS, GENESIS_IP, "tar xvzf ${BUNDLE_DIR}.tar.gz")
  139. timeout (90) {
  140. ssh.cmd(GENESIS_CREDS, GENESIS_IP,
  141. "sudo ${BUNDLE_DIR}/genesis.sh")
  142. // fixme: there is notable initial slowness likely due to coredns
  143. // going out of service and taking time to recover
  144. // this is a long time issue and needs to be taken look at
  145. retry(2) {
  146. ssh.cmd(GENESIS_CREDS, GENESIS_IP,
  147. "sudo -S ${BUNDLE_DIR}/validate-genesis.sh")
  148. }
  149. }
  150. sleep 120 // wait for k8s to calm down
  151. } catch (err) {
  152. debug_report()
  153. error(err)
  154. }
  155. }
  156. }
  157. def shipyard_deploy = { action ->
  158. try {
  159. uuid = UUID.randomUUID().toString()
  160. def req = keystone.retrieveToken(SHIPYARD_CREDS, KEYSTONE_URL)
  161. def token = req.getHeaders()["X-Subject-Token"][0]
  162. shipyard2.uploadConfig(uuid, token, SHIPYARD_URL, SITE_NAME)
  163. shipyard2.waitAction(action: action,
  164. uuid: uuid,
  165. shipyardUrl: SHIPYARD_URL,
  166. keystoneCreds: SHIPYARD_CREDS,
  167. keystoneUrl: KEYSTONE_URL)
  168. } catch (err) {
  169. print err
  170. debug_report()
  171. throw err
  172. }
  173. }
  174. //// uplift utils
  175. def uplift_versions = {
  176. stage('Uplift Versions') {
  177. sh 'sudo apt-get install python3-yaml python3-git -y'
  178. def cmd = 'tools/updater.py --in-file global/software/config/versions.yaml --tag-filter ubuntu_xenial'
  179. if (!UPLIFT_BLACKLIST.isEmpty()) {
  180. cmd += " --skip ${UPLIFT_BLACKLIST}"
  181. }
  182. sh cmd
  183. sh 'git diff'
  184. }
  185. }
  186. def uplift_review = {
  187. withCredentials([sshUserPrivateKey(credentialsId: 'jenkins-uplifter-key',
  188. keyFileVariable: 'SSH_KEY',
  189. usernameVariable: 'SSH_USER')]) {
  190. sh 'sudo apt-get install git-review -y'
  191. sh "cp ${SSH_KEY} ~/.ssh/id_rsa"
  192. sh "ssh-keyscan -p 29418 review.opendev.org >> ~/.ssh/known_hosts"
  193. sh "git clone ssh://${SSH_USER}@review.opendev.org:29418/airship/treasuremap"
  194. sh "scp -p -P 29418 ${SSH_USER}@review.opendev.org:hooks/commit-msg treasuremap/.git/hooks/"
  195. sh "cp global/software/config/versions.yaml treasuremap/global/software/config/versions.yaml"
  196. dir ('treasuremap') {
  197. sh "git config --global user.name 'Jenkins Uplifter'"
  198. sh "git config --global user.email ${SSH_USER}@gmail.com"
  199. sh "git config --global gitreview.username ${SSH_USER}"
  200. sh 'git checkout -b versions/uplift'
  201. sh 'git add global/software/config/versions.yaml'
  202. sh 'git status'
  203. sh 'git commit -m "Auto chart/image uplift to latest"'
  204. sh 'git review -s'
  205. sh 'git review'
  206. }
  207. }
  208. }
  209. //// test utils
  210. def sanity_tests = {
  211. stage('Sanity Tests') {
  212. sh 'sudo apt-get install nmap -y'
  213. withEnv(['TERM_OPTS=-i',
  214. 'OSH_EXT_SUBNET=',
  215. 'OSH_BR_EX_ADDR=',
  216. 'OSH_KEYSTONE_URL=https://identity-sw.atlantafoundry.com/v3',
  217. 'OSH_REGION_NAME=seaworthy',
  218. 'OSH_ADMIN_PASSWD=password123']) {
  219. sh 'tools/tests.sh'
  220. }
  221. }
  222. }
  223. //// main flow
  224. vm(image: 'ubuntu-16.04-server-cloudimg-amd64',
  225. timeout: 360,
  226. publicNet: 'foundry',
  227. buildType: 'flat',
  228. initScript: 'cloud-config',
  229. artifactoryLogs: ARTIFACTORY_LOGS.toBoolean()) {
  230. // wait and make sure genesis is up
  231. ssh.wait (GENESIS_CREDS, GENESIS_IP, 'hostname')
  232. // disable docker/kubelet services
  233. // this is done to be able to properly cleanup genesis after reboot
  234. ssh.cmd (GENESIS_CREDS, GENESIS_IP,
  235. 'sudo systemctl disable kubelet')
  236. ssh.cmd (GENESIS_CREDS, GENESIS_IP,
  237. 'sudo systemctl disable docker')
  238. ssh.cmd (GENESIS_CREDS, GENESIS_IP,
  239. 'sudo touch /forcefsck')
  240. reset_bare_metal()
  241. sh 'sudo apt-get update'
  242. sh 'sudo apt-get install docker.io -y'
  243. clone(AIRSHIP_MANIFESTS_REF)
  244. // use updater tool to pull latest charts/images
  245. if (AIRSHIP_MANIFESTS_REF == 'uplift') {
  246. uplift_versions()
  247. }
  248. pegleg_site_collect()
  249. prom_config_gen()
  250. stage ('Genesis Wait') {
  251. ssh.wait (GENESIS_CREDS, GENESIS_IP, 'hostname')
  252. }
  253. genesis_cleanup()
  254. genesis_deploy()
  255. timeout(240) {
  256. shipyard_deploy('deploy_site')
  257. }
  258. sanity_tests()
  259. if (AIRSHIP_MANIFESTS_REF == 'uplift') {
  260. uplift_review()
  261. }
  262. }