Reference Airship manifests, CICD, and reference architecture.
Ви не можете вибрати більше 25 тем Теми мають розпочинатися з літери або цифри, можуть містити дефіси (-) і не повинні перевищувати 35 символів.

358 рядки
9.6KB

  1. // Pipeline expects genesis to be setup according to the documentation
  2. // including networks, disks, ntp, ip rules, etc.. manually.
  3. // https://airship-treasuremap.readthedocs.io/en/latest/authoring_and_deployment.html#genesis-node
  4. // shared libaries used within the pipeline
  5. // https://github.com/att-comdev/cicd/blob/master/vars
  6. import org.yaml.snakeyaml.Yaml
  7. import org.yaml.snakeyaml.DumperOptions;
  8. import groovy.json.JsonSlurperClassic
  9. import groovy.json.JsonOutput
  10. COLLECT_DIR = 'seaworthy'
  11. BUNDLE_DIR = 'bundle'
  12. KEYSTONE_URL = 'https://iam-sw.atlantafoundry.com'
  13. SHIPYARD_URL = 'https://shipyard-sw.atlantafoundry.com/api/v1.0'
  14. SHIPYARD_CREDS = 'seaworthy-shipyard-iam-pw'
  15. SITE_NAME='seaworthy'
  16. IPMI_CREDS = 'seaworthy-ipmi'
  17. GENESIS_IP = '10.23.21.11'
  18. GENESIS_CREDS = 'seaworthy-key'
  19. GENESIS_IPMI_IP = '10.23.104.11'
  20. GENESIS_CEPH_DISKS = ['b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k']
  21. IPMI_IPS = ['10.23.104.12',
  22. '10.23.104.13',
  23. '10.23.104.14',
  24. '10.23.104.16',
  25. '10.23.104.17']
  26. AIRSHIP_MANIFESTS_REPO = 'https://review.opendev.org/airship/treasuremap'
  27. if (env.GERRIT_REFSPEC) {
  28. AIRSHIP_MANIFESTS_REF = GERRIT_REFSPEC
  29. DISPLAY_NAME = GERRIT_EVENT_TYPE
  30. } else if (AIRSHIP_MANIFESTS_REF == 'uplift') {
  31. DISPLAY_NAME = 'uplift'
  32. } else {
  33. DISPLAY_NAME = "manual ${AIRSHIP_MANIFESTS_REF}"
  34. }
  35. currentBuild.displayName = "#${BUILD_NUMBER} ${DISPLAY_NAME}"
  36. //// git utils
  37. def clone(String ref){
  38. def refspec = ''
  39. // override refspec if patchset provided
  40. if (ref.contains('refs')) {
  41. refspec = "${ref}:${ref}"
  42. }
  43. // base uplift on latest master
  44. if (ref == 'uplift') {
  45. ref = 'master'
  46. }
  47. checkout poll: false,
  48. scm: [$class: 'GitSCM',
  49. branches: [[name: ref]],
  50. doGenerateSubmoduleConfigurations: false,
  51. extensions: [[$class: 'CleanBeforeCheckout']],
  52. submoduleCfg: [],
  53. userRemoteConfigs: [[refspec: refspec,
  54. url: AIRSHIP_MANIFESTS_REPO ]]]
  55. }
  56. //// bare-metal utils
  57. def reset_bare_metal = {
  58. stage ('Bare-Metal Reset') {
  59. withCredentials([usernamePassword(credentialsId: IPMI_CREDS,
  60. usernameVariable: 'IUSER',
  61. passwordVariable: 'IPASS')]) {
  62. def auth = redfish.getBasicAuth(IUSER, IPASS)
  63. // power cycle genesis
  64. redfish.powerOff(GENESIS_IPMI_IP, auth)
  65. redfish.powerOn(GENESIS_IPMI_IP, auth)
  66. // shutdown all other nodes
  67. IPMI_IPS.each() {
  68. redfish.powerOff(it, auth)
  69. }
  70. }
  71. }
  72. }
  73. //// manifest utils
  74. def pegleg_site_collect = {
  75. stage('Pegleg Site Collect') {
  76. configFileProvider([configFile(fileId: 'seaworthy-site-definition',
  77. targetLocation: 'site/seaworthy/site-definition.yaml')]) {
  78. }
  79. withCredentials([sshUserPrivateKey(credentialsId: 'jenkins-attcomdev-key',
  80. keyFileVariable: 'SSH_KEY',
  81. usernameVariable: 'SSH_USER')]) {
  82. sh "cp ${SSH_KEY} ssh-key"
  83. sh "sudo -E tools/airship pegleg site" +
  84. " -u ${SSH_USER}" +
  85. " -k /target/ssh-key" +
  86. " -r /target collect ${SITE_NAME}" +
  87. " -s /target/${COLLECT_DIR}"
  88. sh "sudo chown -R ubuntu:ubuntu ${COLLECT_DIR}"
  89. }
  90. sh "tar czf ${COLLECT_DIR}.tar.gz ${COLLECT_DIR}"
  91. archiveArtifacts "${COLLECT_DIR}.tar.gz"
  92. }
  93. }
  94. def prom_config_gen = {
  95. stage ("Promenade Config Gen") {
  96. withEnv(['TERM_OPTS=-t']) {
  97. sh "mkdir -p ${BUNDLE_DIR}"
  98. sh "sudo -E tools/airship promenade build-all --validators" +
  99. " -o ${BUNDLE_DIR} /target/${COLLECT_DIR}/*.yaml"
  100. }
  101. sh "tar czf ${BUNDLE_DIR}.tar.gz ${BUNDLE_DIR}"
  102. archiveArtifacts "${BUNDLE_DIR}.tar.gz"
  103. }
  104. }
  105. //// genesis utils
  106. def genesis_cleanup = {
  107. stage('Genesis Cleanup') {
  108. dfiles = ['promenade',
  109. BUNDLE_DIR,
  110. "${BUNDLE_DIR}.tar.gz",
  111. SITE_NAME,
  112. 'seaworthy.tar.gz',
  113. 'debug-cab23-r720-11.tgz',
  114. '/var/lib/docker',
  115. '/var/lib/kubelet']
  116. dfiles.each() {
  117. ssh.cmd (GENESIS_CREDS, GENESIS_IP, "sudo rm -rf ${it}")
  118. }
  119. ssh.cmd (GENESIS_CREDS, GENESIS_IP,
  120. 'git clone https://opendev.org/airship/promenade')
  121. ssh.cmd (GENESIS_CREDS, GENESIS_IP,
  122. 'sudo -S promenade/tools/cleanup.sh -f')
  123. GENESIS_CEPH_DISKS.each() {
  124. ssh.cmd(GENESIS_CREDS, GENESIS_IP,
  125. "sudo parted -s /dev/sd${it} mklabel gpt")
  126. }
  127. }
  128. }
  129. def debug_report = {
  130. ssh.cmd (GENESIS_CREDS, GENESIS_IP, 'sudo debug-report.sh')
  131. ssh.get (GENESIS_CREDS, GENESIS_IP, 'debug-cab23-r720-11.tgz', '.')
  132. archiveArtifacts 'debug-cab23-r720-11.tgz'
  133. }
  134. def genesis_deploy = {
  135. stage('Genesis Deploy') {
  136. try {
  137. ssh.put(GENESIS_CREDS, GENESIS_IP, "${BUNDLE_DIR}.tar.gz", '.')
  138. ssh.cmd(GENESIS_CREDS, GENESIS_IP, "tar xvzf ${BUNDLE_DIR}.tar.gz")
  139. timeout (90) {
  140. ssh.cmd(GENESIS_CREDS, GENESIS_IP,
  141. "sudo ${BUNDLE_DIR}/genesis.sh")
  142. // fixme: there is notable initial slowness likely due to coredns
  143. // going out of service and taking time to recover
  144. // this is a long time issue and needs to be taken look at
  145. retry(2) {
  146. ssh.cmd(GENESIS_CREDS, GENESIS_IP,
  147. "sudo -S ${BUNDLE_DIR}/validate-genesis.sh")
  148. }
  149. }
  150. sleep 120 // wait for k8s to calm down
  151. } catch (err) {
  152. debug_report()
  153. error(err)
  154. }
  155. }
  156. }
  157. def shipyard_deploy = { action ->
  158. try {
  159. uuid = UUID.randomUUID().toString()
  160. def req = keystone.retrieveToken(SHIPYARD_CREDS, KEYSTONE_URL)
  161. def token = req.getHeaders()["X-Subject-Token"][0]
  162. shipyard2.uploadConfig(uuid, token, SHIPYARD_URL, SITE_NAME)
  163. shipyard2.waitAction(action: action,
  164. uuid: uuid,
  165. shipyardUrl: SHIPYARD_URL,
  166. keystoneCreds: SHIPYARD_CREDS,
  167. keystoneUrl: KEYSTONE_URL)
  168. } catch (err) {
  169. print err
  170. debug_report()
  171. throw err
  172. }
  173. }
  174. //// uplift utils
  175. def uplift_versions = {
  176. stage('Uplift Versions') {
  177. sh 'sudo apt-get install python3-yaml python3-git -y'
  178. def cmd = 'tools/updater.py --in-file global/software/config/versions.yaml --tag-filter ubuntu_xenial'
  179. if (!UPLIFT_BLACKLIST.isEmpty()) {
  180. cmd += " --skip ${UPLIFT_BLACKLIST}"
  181. }
  182. sh cmd
  183. sh 'git diff'
  184. }
  185. }
  186. def uplift_review = {
  187. withCredentials([sshUserPrivateKey(credentialsId: 'jenkins-uplifter-key',
  188. keyFileVariable: 'SSH_KEY',
  189. usernameVariable: 'SSH_USER')]) {
  190. sh 'sudo apt-get install git-review -y'
  191. sh "cp ${SSH_KEY} ~/.ssh/id_rsa"
  192. sh "ssh-keyscan -p 29418 review.opendev.org >> ~/.ssh/known_hosts"
  193. sh "git clone ssh://${SSH_USER}@review.opendev.org:29418/airship/treasuremap"
  194. sh "scp -p -P 29418 ${SSH_USER}@review.opendev.org:hooks/commit-msg treasuremap/.git/hooks/"
  195. sh "cp global/software/config/versions.yaml treasuremap/global/software/config/versions.yaml"
  196. dir ('treasuremap') {
  197. sh "git config --global user.name 'Jenkins Uplifter'"
  198. sh "git config --global user.email ${SSH_USER}@gmail.com"
  199. sh "git config --global gitreview.username ${SSH_USER}"
  200. sh 'git checkout -b versions/uplift'
  201. sh 'git add global/software/config/versions.yaml'
  202. sh 'git status'
  203. sh 'git commit -m "Auto chart/image uplift to latest"'
  204. sh 'git review -s'
  205. sh 'git review'
  206. }
  207. }
  208. }
  209. //// test utils
  210. def sanity_tests = {
  211. stage('Sanity Tests') {
  212. sh 'sudo apt-get install nmap -y'
  213. withEnv(['TERM_OPTS=-i',
  214. 'OSH_EXT_SUBNET=',
  215. 'OSH_BR_EX_ADDR=',
  216. 'OSH_KEYSTONE_URL=https://identity-sw.atlantafoundry.com/v3',
  217. 'OSH_REGION_NAME=seaworthy',
  218. 'OSH_ADMIN_PASSWD=password123']) {
  219. sh 'tools/tests.sh'
  220. }
  221. }
  222. }
  223. //// main flow
  224. vm(image: 'ubuntu-16.04-server-cloudimg-amd64',
  225. timeout: 360,
  226. publicNet: 'foundry',
  227. buildType: 'flat',
  228. initScript: 'cloud-config',
  229. artifactoryLogs: ARTIFACTORY_LOGS.toBoolean()) {
  230. // wait and make sure genesis is up
  231. ssh.wait (GENESIS_CREDS, GENESIS_IP, 'hostname')
  232. // disable docker/kubelet services
  233. // this is done to be able to properly cleanup genesis after reboot
  234. ssh.cmd (GENESIS_CREDS, GENESIS_IP,
  235. 'sudo systemctl disable kubelet')
  236. ssh.cmd (GENESIS_CREDS, GENESIS_IP,
  237. 'sudo systemctl disable docker')
  238. ssh.cmd (GENESIS_CREDS, GENESIS_IP,
  239. 'sudo touch /forcefsck')
  240. reset_bare_metal()
  241. sh 'sudo apt-get update'
  242. sh 'sudo apt-get install docker.io -y'
  243. clone(AIRSHIP_MANIFESTS_REF)
  244. // use updater tool to pull latest charts/images
  245. if (AIRSHIP_MANIFESTS_REF == 'uplift') {
  246. uplift_versions()
  247. }
  248. pegleg_site_collect()
  249. prom_config_gen()
  250. stage ('Genesis Wait') {
  251. ssh.wait (GENESIS_CREDS, GENESIS_IP, 'hostname')
  252. }
  253. genesis_cleanup()
  254. genesis_deploy()
  255. timeout(240) {
  256. shipyard_deploy('deploy_site')
  257. }
  258. sanity_tests()
  259. if (AIRSHIP_MANIFESTS_REF == 'uplift') {
  260. uplift_review()
  261. }
  262. }