From 7f70ca7ff5ccec53fa17cdf4cc64c52a73168c5e Mon Sep 17 00:00:00 2001 From: Jesse Pretorius Date: Wed, 25 May 2016 16:54:14 +0100 Subject: [PATCH] Set AIO to use an OpenStack-Infra wheel mirror To speed up an AIO build when *not* executed in the OpenStack-CI gate, this patch determines the fastest available wheel mirror and includes it as an extra pip index. This speeds up the repo build process by ensuring that the wheels do not need to be built locally, but can simply be downloaded from the mirror. The patch also includes some clean up which was left out of I73fa1db5210f02d6df9dc324f8b4ec21232d06ba relating to apt conf files. Change-Id: I56f7c8520cc6cec74df5ca8611d988039184efbb --- scripts/fastest-infra-wheel-mirror.py | 170 ++++++++++++++++++ .../tasks/prepare_aio_config.yml | 32 ++-- .../templates/user_variables.aio.yml.j2 | 5 + 3 files changed, 194 insertions(+), 13 deletions(-) create mode 100755 scripts/fastest-infra-wheel-mirror.py diff --git a/scripts/fastest-infra-wheel-mirror.py b/scripts/fastest-infra-wheel-mirror.py new file mode 100755 index 0000000000..4c75acfa79 --- /dev/null +++ b/scripts/fastest-infra-wheel-mirror.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python +# +# Copyright 2016, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2016, Jesse Pretorius +# +# Based on the mirror test script posted at +# http://code.activestate.com/recipes/284631-a-python-script-to-test-download-mirrors/ + +import platform +import Queue +import re +import threading +import time +import urllib + +HTTP_TIMEOUT = 10.0 # Max. seconds to wait for a response +HTTP_TITLE = "Wheel Index" # HTTP Title to look for to validate the page +MAX_THREADS = 10 +MIRROR_LIST = ["http://mirror.dfw.rax.openstack.org/wheel/", + "http://mirror.ord.rax.openstack.org/wheel/", + "http://mirror.iad.rax.openstack.org/wheel/", + "http://mirror.gra1.ovh.openstack.org/wheel/", + "http://mirror.bhs1.ovh.openstack.org/wheel/", + "http://mirror.sjc1.bluebox.openstack.org/wheel/", + "http://mirror.nyj01.internap.openstack.org/wheel/", + "http://mirror.cloud1.osic.openstack.org/wheel/"] + + +def TestUrl(workQueue, resultQueue): + + '''Worker thread procedure. + + Test how long it takes to return the mirror index page, + then return the results into resultQueue. + ''' + + def SubthreadProc(url, result): + + '''Subthread procedure. + + Actually get the mirror index page in a subthread, so that we can time + out using join rather than wait for a very slow server. Passing in a + list for result lets us simulate pass-by-reference, since callers + cannot get the return code from a Python thread. + ''' + + startTime = time.time() + try: + data = urllib.urlopen(url).read() + except Exception: + # Could be a socket error or an HTTP error--either way, we + # don't care--it's a failure to us. + result.append(-1) + else: + if not CheckTitle(data): + result.append(-1) + else: + elapsed = int((time.time() - startTime) * 1000) + result.append(elapsed) + + def CheckTitle(html): + + '''Check that the HTML title is the expected value. + + Check the HTML returned for the presence of a specified + title. This caters for a situation where a service provider + may be redirecting DNS resolution failures to a web search + page, or where the returned data is invalid in some other + way. + ''' + + titleRegex = re.compile("(.+?)") + try: + title = titleRegex.search(html).group(1) + except Exception: + # If there is no match, then we consider it a failure. + result.append(-1) + else: + if title == HTTP_TITLE: + return True + else: + return False + + while 1: + # Continue pulling data from the work queue until it's empty + try: + url = workQueue.get(0) + except Queue.Empty: + # work queue is empty--exit the thread proc. + return + + # Create a single subthread to do the actual work + result = [] + subThread = threading.Thread(target=SubthreadProc, args=(url, result)) + + # Daemonize the subthread so that even if a few are hanging + # around when the process is done, the process will exit. + subThread.setDaemon(True) + + # Run the subthread and wait for it to finish, or time out + subThread.start() + subThread.join(HTTP_TIMEOUT) + + if [] == result: + # Subthread hasn't give a result yet. Consider it timed out. + resultQueue.put((url, "TIMEOUT")) + elif -1 == result[0]: + # Subthread returned an error from geturl. + resultQueue.put((url, "FAILED")) + else: + # Subthread returned a time. Store it. + resultQueue.put((url, result[0])) + +# Set the number of threads to use +numThreads = min(MAX_THREADS, len(MIRROR_LIST)) + +# Build a queue to feed the worker threads +workQueue = Queue.Queue() +for url in MIRROR_LIST: + # Build the complete URL + distro = platform.linux_distribution()[0].lower() + version = platform.linux_distribution()[1] + architecture = platform.machine() + fullUrl = url + distro + "-" + version + "-" + architecture + "/" + workQueue.put(fullUrl) + +workers = [] +resultQueue = Queue.Queue() + +# Create worker threads to load-balance the retrieval +for threadNum in range(0, numThreads): + workers.append(threading.Thread(target=TestUrl, + args=(workQueue, resultQueue))) + workers[-1].start() + +# Wait for all the workers to finish +for w in workers: + w.join() + +# Separate the successes from failures +timings = [] +failures = [] +while not resultQueue.empty(): + url, result = resultQueue.get(0) + if isinstance(result, str): + failures.append((result, url)) + else: + timings.append((result, url)) + +# Sort by increasing time or result string +timings.sort() +failures.sort() + +# If all results are failed, then exit silently +if len(timings) > 0: + # Print out the fastest mirror URL + print(timings[0][1]) diff --git a/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml b/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml index 5308c6f430..2371c089ee 100644 --- a/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml +++ b/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml @@ -136,6 +136,21 @@ tags: - set-fact-lxc_cache_resolvers +- name: Determine if the host has a global pip config file + stat: + path: /etc/pip.conf + register: pip_conf_file + +- name: Determine the fastest available OpenStack-Infra wheel mirror + shell: ../scripts/fastest-infra-wheel-mirror.py + register: fastest_wheel_mirror + when: not pip_conf_file.stat.exists + +- name: Set repo_build_pip_extra_indexes fact + set_fact: + repo_build_pip_extra_indexes: "[\"{{ fastest_wheel_mirror.stdout }}\"]" + when: not pip_conf_file.stat.exists + - name: Set the user_variables config_template: src: user_variables.aio.yml.j2 @@ -143,17 +158,11 @@ config_overrides: "{{ user_variables_overrides | default({}) }}" config_type: yaml -- name: Determine if the host has a global pip config file - stat: - path: /etc/pip.conf - register: pip_conf_file - - name: Add user_conf_files to contain the list of files to copy into containers file: path: /etc/openstack_deploy/user_conf_files.yml state: touch - when: - - apt_conf_files is defined or pip_conf_file.stat.exists + when: pip_conf_file.stat.exists tags: - container-conf-files @@ -162,8 +171,7 @@ dest: /etc/openstack_deploy/user_conf_files.yml line: "---" insertbefore: BOF - when: - - apt_conf_files is defined or pip_conf_file.stat.exists + when: pip_conf_file.stat.exists tags: - container-conf-files @@ -172,8 +180,7 @@ dest: /etc/openstack_deploy/user_conf_files.yml line: "lxc_container_cache_files:" insertafter: "^---" - when: - - apt_conf_files is defined or pip_conf_file.stat.exists + when: pip_conf_file.stat.exists tags: - container-conf-files @@ -181,7 +188,6 @@ lineinfile: dest: /etc/openstack_deploy/user_conf_files.yml line: " - { src: '/etc/pip.conf', dest: '/etc/pip.conf' }" - when: - - pip_conf_file.stat.exists + when: pip_conf_file.stat.exists tags: - container-conf-files diff --git a/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 b/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 index 8b1527ac42..bf5d0ca563 100644 --- a/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 +++ b/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 @@ -93,3 +93,8 @@ lxc_cache_resolvers: {{ lxc_cache_resolvers }} ## Security hardening apply_security_hardening: true + +{% if repo_build_pip_extra_indexes is defined %} +## Wheel mirrors for the repo_build to use +repo_build_pip_extra_indexes: {{ repo_build_pip_extra_indexes }} +{% endif %}