Merge pull request #426 from cloudnull/pip-packaging-updates

Updated pip wheel building and storage process.

rpc_deployment/roles/common/tasks/install_pip_deps.yml

@@ -16,7 +16,7 @@
 - name: Install pip dependencies
   pip:
     name: "{{ item }}"
-    extra_args: "--allow-all-external"
+    extra_args: "{{ pip_install_options|default('') }}"
   with_items: service_pip_dependencies
   when: service_pip_dependencies is defined
   register: pip_install

rpc_deployment/roles/common/templates/pip/base/global.conf.j2

@@ -2,3 +2,7 @@
 no-index = true
 pre = true
 timeout = 120
+
+[install]
+upgrade = true
+allow_all_external = true

rpc_deployment/roles/openstack_common/tasks/pip_install.yml

@@ -16,7 +16,7 @@
 - name: Install service source
   pip:
     name: "/opt/{{ service_name }}_{{ git_install_branch | replace('/', '_') }}"
-    extra_args: "{{ pip_install_options|default('') }} --allow-all-external"
+    extra_args: "{{ pip_install_options|default('') }}"
   register: pip_install
   until: pip_install|success
   retries: 5
@@ -27,7 +27,7 @@
 - name: Install pip repo plugins
   pip:
     name: "{{ git_dest }}/{{ item.path }}/{{ item.package }}"
-    extra_args: "{{ pip_install_options|default('') }} --allow-all-external"
+    extra_args: "{{ pip_install_options|default('') }}"
   when: git_dest is defined and git_repo_plugins is defined
   with_items: git_repo_plugins
   register: pip_install

scripts/rpc-wheel-builder.py

@@ -38,7 +38,8 @@ PYTHON_PACKAGES = {
     'base_release': dict(),
     'known_release': dict(),
     'from_git': dict(),
-    'required_packages': dict()
+    'required_packages': dict(),
+    'built_files': list()
 }
 
 GIT_REPOS = []
@@ -54,28 +55,31 @@ VERSION_DESCRIPTORS = [
     '>=', '<=', '==', '!=', '<', '>'
 ]
 
+# Defines constant for use later.
+LOG = None
+
 
 class IndicatorThread(object):
     """Creates a visual indicator while normally performing actions."""
 
-    def __init__(self, work_q=None, system=True, debug=False):
+    def __init__(self, note=None, system=True, debug=False, quiet=False):
-        """System Operations Available on Load.
+        """System Operations Available on Load."""
-
-        :param work_q:
-        :param system:
-        """
 
+        self.quiet = quiet
         self.debug = debug
-        self.work_q = work_q
         self.system = system
+        self.note = note
+        if self.note is None:
+            self.note = 'Please Wait... '
         self.job = None
 
     def __enter__(self):
-        if self.debug is False:
+        if all([self.debug is False, self.quiet is False]):
-            self.indicator_thread()
+            return self.indicator_thread()
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        if self.debug is False:
+        self.system = False
+        if all([self.debug is False, self.quiet is False]):
             print('Done.')
             self.job.terminate()
 
@@ -85,7 +89,7 @@ class IndicatorThread(object):
         while self.system:
             busy_chars = ['|', '/', '-', '\\']
             for bc in busy_chars:
-                note = 'Please Wait... '
+                note = self.note
                 sys.stdout.write('\rProcessing - [ %s ] - %s' % (bc, note))
                 sys.stdout.flush()
                 time.sleep(.1)
@@ -113,7 +117,7 @@ def get_file_names(path, ext=None):
     """
 
     paths = os.walk(os.path.abspath(path))
-    files = []
+    files = list()
     for fpath, _, afiles in paths:
         for afile in afiles:
             if ext is not None:
@@ -301,7 +305,7 @@ def retryloop(attempts, timeout=None, delay=None, backoff=1, obj=None):
     _error_handler(msg=error)
 
 
-def build_wheel(wheel_dir, build_dir, dist=None, pkg_name=None, quiet=False,
+def build_wheel(wheel_dir, build_dir, link_dir, dist=None, pkg_name=None,
                 make_opts=None):
     """Execute python wheel build command.
 
@@ -314,7 +318,7 @@ def build_wheel(wheel_dir, build_dir, dist=None, pkg_name=None, quiet=False,
         'pip',
         'wheel',
         '--find-links',
-        wheel_dir,
+        link_dir,
         '--timeout',
         '120',
         '--wheel-dir',
@@ -337,24 +341,36 @@ def build_wheel(wheel_dir, build_dir, dist=None, pkg_name=None, quiet=False,
 
     build_command = ' '.join(command)
     LOG.info('Command: %s' % build_command)
+    output, unused_err = None, None
     for retry in retryloop(3, obj=build_command, delay=2, backoff=1):
         try:
-            with IndicatorThread(debug=quiet):
+            process = subprocess.Popen(
-                ret_data = subprocess.check_call(
                 command,
-                    stdout=LoggerWriter(),
+                stdout=subprocess.PIPE,
                 stderr=LoggerWriter()
             )
+            output, unused_err = process.communicate()
+            retcode = process.poll()
 
-            LOG.info('Command return code: [ %s ]', ret_data)
+            LOG.info('Command return code: [ %s ]', retcode)
-            if ret_data:
+            if retcode:
-                raise subprocess.CalledProcessError(ret_data, build_command)
+                raise subprocess.CalledProcessError(
+                    retcode, build_command, output=output
+                )
         except subprocess.CalledProcessError as exp:
             LOG.warn(
-                'Process failure. Error: [ %s ]. Removing build directory'
+                'Process failure. stderr: [ %s ], stdout [ %s ], Exception'
-                ' for retry. Check log for more detauls.', str(exp)
+                ' [ %s ]. Removing build directory for retry. Check log for'
+                ' more details.',
+                unused_err,
+                output,
+                str(exp)
             )
+            remove_dirs(directory=build_dir)
             retry()
+        finally:
+            # Ensure the build directories are clean
+            remove_dirs(directory=build_dir)
 
 
 def remove_dirs(directory):
@@ -380,73 +396,117 @@ def remove_dirs(directory):
             pass
 
 
-def _requirements_maker(name, wheel_dir, release, build_dir, quiet, make_opts,
+def copy_file(src, dst):
-                        iterate=False):
+    LOG.debug('Copying [ %s ] -> [ %s ]', src, dst)
-    requirements_file_lines = []
+    with open(src, 'rb') as open_src:
-    for value in sorted(release.values()):
+        with open(dst, 'wb') as open_dst:
-        requirements_file_lines.append('%s\n' % value)
+            while True:
+                buf = open_src.read(24 * 1024)
+                if not buf:
+                    break
+                else:
+                    open_dst.write(buf)
 
-    requirements_file = os.path.join(wheel_dir, name)
+
-    with open(requirements_file, 'wb') as f:
+def _requirements_maker(name, wheel_dir, release, build_dir, make_opts,
-        f.writelines(requirements_file_lines)
+                        link_dir=None, iterate=False):
+
+    if link_dir is None:
+        link_dir = wheel_dir
 
     if iterate is True:
         for pkg in sorted(release.values()):
             build_wheel(
                 wheel_dir=wheel_dir,
                 build_dir=build_dir,
-                dist=None,
+                link_dir=link_dir,
                 pkg_name=pkg,
-                quiet=quiet,
                 make_opts=make_opts
             )
-            remove_dirs(directory=build_dir)
     else:
+        requirements_file_lines = []
+        for value in sorted(set(release.values())):
+            requirements_file_lines.append('%s\n' % value)
+
+        requirements_file = os.path.join(wheel_dir, name)
+        with open(requirements_file, 'wb') as f:
+            f.writelines(requirements_file_lines)
+
         build_wheel(
             wheel_dir=wheel_dir,
             build_dir=build_dir,
+            link_dir=link_dir,
             dist=requirements_file,
-            quiet=quiet,
             make_opts=make_opts
         )
-        remove_dirs(directory=build_dir)
 
 
-def make_wheels(wheel_dir, build_dir, quiet):
+def _make_wheels(wheel_dir, build_dir, temp_store_dir):
+    LOG.info('Building base packages')
+    _requirements_maker(
+        name='rpc_base_requirements.txt',
+        wheel_dir=temp_store_dir,
+        release=PYTHON_PACKAGES['base_release'],
+        build_dir=build_dir,
+        make_opts=None,
+        link_dir=wheel_dir
+    )
+
+    LOG.info('Building known absolute packages')
+    _requirements_maker(
+        name='rpc_known_requirements.txt',
+        wheel_dir=temp_store_dir,
+        release=PYTHON_PACKAGES['known_release'],
+        build_dir=build_dir,
+        make_opts=['--no-deps'],
+        link_dir=wheel_dir
+    )
+
+    LOG.info('Building required packages')
+    _requirements_maker(
+        name='rpc_required_requirements.txt',
+        wheel_dir=temp_store_dir,
+        release=PYTHON_PACKAGES['required_packages'],
+        build_dir=build_dir,
+        make_opts=None,
+        link_dir=wheel_dir,
+        iterate=True
+    )
+
+    built_wheels = get_file_names(temp_store_dir)
+    PYTHON_PACKAGES['built_files'] = [
+        os.path.basename(i) for i in built_wheels
+    ]
+
+    LOG.info('Moving built packages into place')
+    for built_wheel in built_wheels:
+        wheel_file = os.path.join(wheel_dir, os.path.basename(built_wheel))
+        if os.path.exists(wheel_file):
+            if os.path.getsize(wheel_file) != os.path.getsize(built_wheel):
+                copy_file(src=built_wheel, dst=wheel_file)
+        else:
+            copy_file(src=built_wheel, dst=wheel_file)
+
+
+def make_wheels(wheel_dir, build_dir):
     """Build wheels of all installed packages that don't already have one.
 
     :param wheel_dir: ``str`` $PATH to local save directory
     :param build_dir: ``str`` $PATH to temp build directory
     """
 
-    _requirements_maker(
+    temp_store_dir = os.path.join(
-        name='rpc_base_requirements.txt',
+        tempfile.mkdtemp(prefix='rpc_wheels_temp_storage')
-        wheel_dir=wheel_dir,
-        release=PYTHON_PACKAGES['base_release'],
-        build_dir=build_dir,
-        quiet=quiet,
-        make_opts=None
     )
-
+    _mkdirs(path=temp_store_dir)
-    _requirements_maker(
+    try:
-        name='rpc_required_requirements.txt',
+        _make_wheels(
             wheel_dir=wheel_dir,
-        release=PYTHON_PACKAGES['required_packages'],
             build_dir=build_dir,
-        quiet=quiet,
+            temp_store_dir=temp_store_dir
-        make_opts=None,
-        iterate=True
         )
-
+    finally:
-    _requirements_maker(
+        remove_dirs(directory=temp_store_dir)
-        name='rpc_known_requirements.txt',
-        wheel_dir=wheel_dir,
-        release=PYTHON_PACKAGES['known_release'],
-        build_dir=build_dir,
-        quiet=quiet,
-        make_opts=['--no-deps']
-    )
-
         remove_dirs(
             directory=os.path.join(
                 tempfile.gettempdir(),
@@ -463,7 +523,7 @@ def ensure_consistency():
         PYTHON_PACKAGES['base_release'].pop(key, None)
 
 
-def new_setup(user_args, input_path, output_path, quiet):
+def new_setup(user_args, input_path):
     """Discover all yaml files in the input directory."""
 
     LOG.info('Discovering input file(s)')
@@ -483,35 +543,12 @@ def new_setup(user_args, input_path, output_path, quiet):
 
     # Populate the package dict
     LOG.info('Building the package list')
-    with IndicatorThread(debug=quiet):
     for var_file in var_files:
         package_dict(var_file=var_file)
 
     # Ensure no general packages take precedence over the explicit ones
     ensure_consistency()
 
-    # Get a timestamp and create a report file
-    utctime = datetime.datetime.utcnow()
-    utctime = utctime.strftime("%Y%m%d_%H%M%S")
-    backup_name = 'python-build-report-%s.json' % utctime
-    output_report_file = os.path.join(
-        output_path,
-        'json-reports',
-        backup_name
-    )
-    _mkdirs(os.path.dirname(output_report_file))
-
-    # Generate a timestamped report file
-    LOG.info('Generating packaging report [ %s ]', output_report_file)
-    with open(output_report_file, 'wb') as f:
-        f.write(
-            json.dumps(
-                PYTHON_PACKAGES,
-                indent=2,
-                sort_keys=True
-            )
-        )
-
 
 def _error_handler(msg, system_exit=True):
     """Handle and error logging and exit the application if needed.
@@ -570,6 +607,19 @@ def _user_args():
         required=False,
         default=None
     )
+    parser.add_argument(
+        '--link-dir',
+        help='Path to the build links for all built wheels.',
+        required=False,
+        default=None
+    )
+    parser.add_argument(
+        '-r',
+        '--release',
+        help='Name of the release. Used for generating the json report.',
+        required=True,
+        default=None
+    )
     opts = parser.add_mutually_exclusive_group()
     opts.add_argument(
         '--debug',
@@ -617,16 +667,14 @@ def _mkdirs(path):
             _error_handler(msg=error)
 
 
-def _store_git_repos(git_repos_path, quiet):
+def _store_git_repos(git_repos_path):
     """Clone and or update all git repos.
 
     :param git_repos_path: ``str`` Path to where to store the git repos
-    :param quiet: ``bol`` Enable quiet mode.
     """
     _mkdirs(git_repos_path)
     for retry in retryloop(3, delay=2, backoff=1):
         for git_repo in GIT_REPOS:
-            with IndicatorThread(debug=quiet):
             repo_name = os.path.basename(git_repo)
             if repo_name.endswith('.git'):
                 repo_name = repo_name.rstrip('git')
@@ -696,6 +744,12 @@ def main():
 
     # Create the build path
     LOG.info('Getting build path')
+    indicator_kwargs = {
+        'debug': user_args['debug'],
+        'quiet': user_args['quiet'],
+        'note': 'Gather dependencies... '
+    }
+    with IndicatorThread(**indicator_kwargs):
         if user_args['build_dir'] is not None:
             build_path = _get_abs_path(path=user_args['build_dir'])
             _mkdirs(path=build_path)
@@ -710,26 +764,89 @@ def main():
             else:
                 # Get the input path
                 LOG.info('Getting input path')
-            input_path = _get_abs_path(path=user_args['input'])
                 new_setup(
                     user_args=user_args,
-                input_path=input_path,
+                    input_path=_get_abs_path(path=user_args['input'])
-                output_path=output_path,
-                quiet=stream
                 )
 
+    indicator_kwargs['note'] = 'Building wheels... '
+    with IndicatorThread(**indicator_kwargs):
         # Create all of the python package wheels
         make_wheels(
             wheel_dir=output_path,
-            build_dir=build_path,
+            build_dir=build_path
-            quiet=stream
         )
 
+    indicator_kwargs['note'] = 'Generating build log... '
+    with IndicatorThread(**indicator_kwargs):
+        # Get a timestamp and create a report file
+        utctime = datetime.datetime.utcnow()
+        utctime = utctime.strftime("%Y%m%d_%H%M%S")
+        backup_name = '%s-build-report-%s.json' % (
+            user_args['release'],
+            utctime
+        )
+        output_report_file = os.path.join(
+            output_path,
+            'json-reports',
+            backup_name
+        )
+
+        # Make the directory if needed
+        _mkdirs(path=os.path.dirname(output_report_file))
+
+        # Generate a timestamped report file
+        LOG.info('Generating packaging report [ %s ]', output_report_file)
+        with open(output_report_file, 'wb') as f:
+            f.write(
+                json.dumps(
+                    PYTHON_PACKAGES,
+                    indent=2,
+                    sort_keys=True
+                )
+            )
+
+    # If link_dir is defined create a link to all built wheels.
+    links_path = user_args.get('link_dir')
+    if links_path:
+        indicator_kwargs['note'] = 'Creating file links... '
+        with IndicatorThread(**indicator_kwargs):
+            links_path = _get_abs_path(path=links_path)
+            LOG.info('Creating Links at [ %s ]', links_path)
+            _mkdirs(path=links_path)
+
+            # Change working directory.
+            os.chdir(links_path)
+
+            # Create all the links
+            for inode in PYTHON_PACKAGES['built_files']:
+                try:
+                    dest_link = os.path.join(links_path, inode)
+
+                    # Remove the destination inode if it exists
+                    if os.path.exists(dest_link):
+                        os.remove(dest_link)
+
+                    # Create the link using the relative path
+                    os.symlink(os.path.relpath(
+                        os.path.join(output_path, inode)), dest_link
+                    )
+                except OSError as exp:
+                    LOG.warn(
+                        'Error Creating Link: [ %s ] Error: [ %s ]',
+                        inode,
+                        exp
+                    )
+                else:
+                    LOG.debug('Link Created: [ %s ]', dest_link)
+
     # if git_repos was defined save all of the sources to the defined location
     git_repos_path = user_args.get('git_repos')
     if git_repos_path:
-        _store_git_repos(git_repos_path, quiet=stream)
+        indicator_kwargs['note'] = 'Storing updated git sources...'
-
+        with IndicatorThread(**indicator_kwargs):
+            LOG.info('Updating git sources [ %s ]', links_path)
+            _store_git_repos(_get_abs_path(path=git_repos_path))
 
 
 if __name__ == "__main__":

scripts/rpc-wheel-builder.sh

@@ -13,22 +13,128 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Notes:
+# To use this script you MUST move it to some path that will be called.
+# I recommend that the script be stored and executed from 
+# "/opt/rpc-wheel-builder.sh". This script is a wrapper script that relies
+#  on the "rpc-wheel-builder.py" and is execute from
+# "/opt/rpc-wheel-builder.py".
+
+# Overrides:
+# This script has several things that can be overriden via environment 
+# variables.
+#     Git repository that the rcbops ansible lxc source code will be cloned from. 
+#     This repo should be a repo that is available via HTTP.
+#     GIT_REPO=""
+
+#     The URI for the github api. This is ONLY used when the $RELEASES variable
+#     is an empty string. Which causes the script to go discover the available
+#     releases.
+#     GITHUB_API_ENDPOINT=""
+
+#     Local directory to store the source code while interacting with it.
+#     WORK_DIR=""
+
+#     Local directory to store the built wheels.
+#     OUTPUT_WHEEL_PATH=""
+
+#     Local directory to store known git repos.
+#     OUTPUT_GIT_PATH=""
+
+#     Space seperated list of all releases to build for. If unset the releases
+#     will be discovered.
+#     RELEASES=""
+
+#     Space seperated list of all releases to exclude from building. This is
+#     ONLY used when the $RELEASES variable is an empty string.
+#     EXCLUDE_RELEASES=""
+
 set -e -o -v
 
-WORK_DIR="/opt/ansible-lxc-rpc"
+# Ensure there is a base path loaded
-GIT_REPO="https://github.com/rcbops/ansible-lxc-rpc"
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-REPO_PACKAGES_PATH="/opt/ansible-lxc-rpc/rpc_deployment/vars/repo_packages/"
-OUTPUT_WHEEL_PATH="/var/www/repo/python_packages"
-RELEASE=$1
 
+# Defined variables
+GIT_REPO="${GIT_REPO:-https://github.com/rcbops/ansible-lxc-rpc}"
+GITHUB_API_ENDPOINT="${GITHUB_API_ENDPOINT:-https://api.github.com/repos/rcbops/ansible-lxc-rpc}"
+
+WORK_DIR="${WORK_DIR:-/opt/ansible-lxc-rpc}"
+REPO_PACKAGES_PATH="${WORK_DIR}/rpc_deployment/vars/repo_packages/"
+
+OUTPUT_WHEEL_PATH="${OUTPUT_WHEEL_PATH:-/var/www/repo/python_packages}"
+OUTPUT_GIT_PATH="${OUTPUT_GIT_PATH:-/var/www/repo/rpcgit}"
+
+# Default is an empty string which causes the script to go discover the available
+# branches from the github API.
+RELEASES=${RELEASES:-""}
+
+EXCLUDE_RELEASES="${EXCLUDE_RELEASES:-v9.0.0 gh-pages revert}"
+
+if [[ ! "${RELEASES}" ]];then
+# From the GITHUB API pull a list of all branches/tags
+RELEASES=$(
+$(which python) <<EOF
+import requests
+
+# Create an array of excluded items
+EXCLUDE = "${EXCLUDE_RELEASES}".split()
+
+def return_releases(url):
+    """Return a list of releases found in the github api.
+
+    :param url: ``str``
+    """
+    _releases = requests.get(url)
+    loaded_releases = _releases.json()
+    releases = list()
+    for i in loaded_releases:
+        for k, v in i.iteritems():
+            if k == 'name':
+                # if the name is not excluded append it
+                if not any([v.startswith(i) for i in EXCLUDE]):
+                    releases.append(v)
+    else:
+        # Return a unique list.
+        return list(set(releases))
+
+all_releases = list()
+all_releases.extend(return_releases(url="${GITHUB_API_ENDPOINT}/tags"))
+all_releases.extend(return_releases(url="${GITHUB_API_ENDPOINT}/branches"))
+print(' '.join(all_releases))
+EOF
+)
+fi
+
+
+function cleanup() {
+    # Ensure workspaces are cleaned up
     rm -rf /tmp/rpc_wheels*
     rm -rf /tmp/pip*
     rm -rf "${WORK_DIR}"    
+}
 
+
+# Iterate through the list of releases and build everything that's needed
+for release in ${RELEASES}; do
+
+    # Perform cleanup
+    cleanup
+
+    # Git clone repo
     git clone "${GIT_REPO}" "${WORK_DIR}"
+
+    # checkout release
     pushd "${WORK_DIR}"
-  git checkout "${RELEASE}"
+      git checkout "${release}"
     popd
 
-${WORK_DIR}/scripts/rpc-wheel-builder.py -i "${REPO_PACKAGES_PATH}" \
+    # Build wheels
-                                         -o "${OUTPUT_WHEEL_PATH}"/"${RELEASE}"
+    /opt/rpc-wheel-builder.py -i "${REPO_PACKAGES_PATH}" \
+                              -o "${OUTPUT_WHEEL_PATH}"/pools \
+                              --link-dir "${OUTPUT_WHEEL_PATH}"/"${release}" \
+                              --git-repos "${OUTPUT_GIT_PATH}" \
+                              --release "${release}"
+done
+
+# Perform cleanup
+cleanup