From 622f04f2147adae09ea0c2b59bb95c8bbd1b35d1 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 1 Sep 2017 17:07:15 -0500 Subject: [PATCH] Treat wheel builders more like build nodes Pass the wheel keytab across and aklog with it. Also, split the build role into a build and a copy so that we only have the token when we need it, and not while we're running the wheel builds themselves. And split release from build so that we can release the mirror once for a consistent state across python2 and python3. Change-Id: Id30a99f5b515dd520d013fabbd356e32c82030a0 --- playbooks/wheel/build.yaml | 8 ++--- playbooks/wheel/release.yaml | 4 +-- roles/build-wheels/README.rst | 1 + roles/build-wheels/tasks/main.yaml | 14 -------- roles/copy-wheels/README.rst | 1 + roles/copy-wheels/tasks/main.yaml | 5 +++ zuul.yaml | 54 +++++++++++++++++++++++++----- 7 files changed, 59 insertions(+), 28 deletions(-) create mode 100644 roles/build-wheels/README.rst create mode 100644 roles/copy-wheels/README.rst create mode 100644 roles/copy-wheels/tasks/main.yaml diff --git a/playbooks/wheel/build.yaml b/playbooks/wheel/build.yaml index 3efa9f357f..94f2c40655 100644 --- a/playbooks/wheel/build.yaml +++ b/playbooks/wheel/build.yaml @@ -24,7 +24,7 @@ wheel_dir: /opt/wheel/workspace roles: - - role: build-wheels - wheel_python: python2 - - role: build-wheels - wheel_python: python3 + - build-wheels + - create-afs-token + - copy-wheels + - destroy-afs-token diff --git a/playbooks/wheel/release.yaml b/playbooks/wheel/release.yaml index d6ef3fc0a2..8f0a6d0c64 100644 --- a/playbooks/wheel/release.yaml +++ b/playbooks/wheel/release.yaml @@ -1,10 +1,10 @@ - hosts: localhost roles: - - create-afs-token + - role: create-afs-token - role: release-afs afs_volume: trustyx64 - role: release-afs afs_volume: cent7x64 - role: release-afs afs_volume: xenialx64 - - destroy-afs-token + - role: destroy-afs-token diff --git a/roles/build-wheels/README.rst b/roles/build-wheels/README.rst new file mode 100644 index 0000000000..6e4c38c390 --- /dev/null +++ b/roles/build-wheels/README.rst @@ -0,0 +1 @@ +Run the build scripts on the wheel mirror builders diff --git a/roles/build-wheels/tasks/main.yaml b/roles/build-wheels/tasks/main.yaml index 8d25ad3b29..4dfc37e322 100644 --- a/roles/build-wheels/tasks/main.yaml +++ b/roles/build-wheels/tasks/main.yaml @@ -10,17 +10,3 @@ - name: Build the wheel mirror shell: bash -x scripts/wheel-build.sh {{ wheel_dir }} {{ wheel_python }} - -- name: Get an afs token and copy the wheels to AFS - command: < - k5start -t -f /etc/wheel.keytab - service/wheel - -- timeout -k 2m 30m - scripts/wheel-copy.sh {{ wheel_dir }} {{ afs_dir }} - -- name: Obtaining token and rebuilding mirror index - command: < - k5start -t -f /etc/wheel.keytab - service/wheel - -- timeout -k 2m 30m - scripts/wheel-index.sh {{ afs_dir }} diff --git a/roles/copy-wheels/README.rst b/roles/copy-wheels/README.rst new file mode 100644 index 0000000000..54664828ee --- /dev/null +++ b/roles/copy-wheels/README.rst @@ -0,0 +1 @@ +Run the copy scripts on the wheel mirror builders diff --git a/roles/copy-wheels/tasks/main.yaml b/roles/copy-wheels/tasks/main.yaml new file mode 100644 index 0000000000..0d928f5105 --- /dev/null +++ b/roles/copy-wheels/tasks/main.yaml @@ -0,0 +1,5 @@ +- name: Copy the wheels to AFS + command: bash -x scripts/wheel-copy.sh {{ wheel_dir }} {{ afs_dir }} + +- name: Rebuild mirror index + command: bash -x scripts/wheel-index.sh {{ afs_dir }} diff --git a/zuul.yaml b/zuul.yaml index 3dca1d18d0..d66049fe82 100644 --- a/zuul.yaml +++ b/zuul.yaml @@ -280,13 +280,12 @@ - job: name: build-wheel-mirror + pre-run: playbooks/wheel/pre run: playbooks/wheel/build post-run: playbooks/wheel/release nodes: [] - final: True - # 4.5 hours - double the v2 runtime, since we're running python2 and - # python3 in series - timeout: 16200 + # 2.5 hours + timeout: 9000 allowed-projects: - openstack/requirements semaphore: wheel-mirror @@ -294,9 +293,29 @@ - wheel_mirror_centos_7_key - wheel_mirror_ubuntu_trusty_key - wheel_mirror_ubuntu_xenial_key + - name: afs + secret: wheel_keytab + +- job: + name: release-wheel-mirror + run: playbooks/wheel/release + nodes: [] + secrets: - name: afs secret: afsadmin_keytab +- job: + name: build-wheel-mirror-python2 + parent: build-wheel-mirror + vars: + wheel_python: python2 + +- job: + name: build-wheel-mirror-python3 + parent: build-wheel-mirror + vars: + wheel_python: python3 + - project: name: openstack-infra/project-config check: @@ -577,6 +596,22 @@ gcFWp3Xx4jIQHQdx7I5ZHTLSojvChAf2b8/Ps4IvYTiBKxt7wLtyfV9/kj3BAQ= service_name: service/afsadmin@OPENSTACK.ORG +- secret: + name: wheel_keytab + data: + keytab: !encrypted/pkcs1-oaep + - B12EKQPl+E46VynbYHYNOyzbpIYkxLjia3uDsYjah39m2e/VtrnfprDAk/xOzxqZiC4/u + uG9bnvzHaUZE6BpoUNja1Tr5Mlzk9UY9OGxnb7KGEzIxxyS3s2H6r50FhlSSwK9VZc8fB + Qn2Gp1Vo+Z5VgJoi/W0S7HSPcc9RW+nLdO6S0yX4TLc7EQUlwgQCWowZg5F9V7iouYccz + hJ/ZpM3aAK/77vF70gkX2mXKAtu6/AyCUBnEaEG13KrSiONw95xW7+re1WIuHQxDNGbOH + FInMT0NxDm7hJp8NycY5F5Otwtx1rVOoAK3ReWYKxiTz6JsaM70jpnwc6Ps3eTXwk2B12 + mxBsOtjNXJVVF+Np8lKQ/lk7DweHC0uqM+qbX40er1mcYfT9X+Z8W6lqP3FFgWj3cYqBf + CsTmqIk+W7PaBG6BCe2635L2Ai7gtlAqavo/J+Cry89zsuaj+AkcVzA6impXhNR9bu0Vm + ptwYcxQLEjC+AMUAYwLGhtNGG1wpz1s3kjHyq4ggYUep7C58M3Sk8K58xjVnzAKOldAL1 + EN/khW6Dz+h7qOktic0Nia2AXRG3U8EQY8G/ZL6GGhvlldABqTi7uyLwtUHq+edirlO+5 + rk1ilszKDQOr+gBfweJGrJVcTCTOv+uKf4V8jeSTn6d7Wlu5k3tchI41A3pWeA= + service_name: service/wheel@OPENSTACK.ORG + - secret: name: wheel_mirror_ubuntu_xenial_key data: @@ -754,7 +789,10 @@ periodic: jobs: # This is a periodic job to ensure that our wheels are reasonably - # up-to-date. Adding it to the requirements project seems to be the - # best place to put it, though it's not strictly a requirement for - # requirements. - - build-wheel-mirror + # up-to-date. It uses the upper-constraints files from requirements. + - build-wheel-mirror-python2 + - build-wheel-mirror-python3 + - release-wheel-mirror: + dependencies: + - build-wheel-mirror-python2 + - build-wheel-mirror-python3