Add Dockerfile to build manylinux wheels
To build, $ docker build . --tag pyeclib-build-wheel $ for v in cp27-cp27m cp27-cp27mu cp35-cp35m ; do > docker run --rm --env PYTHON_VERSION=$v --env UID=$UID \ > --env GID=$(id -g) --volume $PWD:/output:Z pyeclib-build-wheel > done It should create x86_64 wheels suitable for CPython 2.7 and 3.5+ that include both liberasurecode and ISA-L libraries. Note that the pack_wheel.py script is useful even without the manylinux Docker container. It can even build self-contained wheels on OS X, though I've only tested on an old x86_64 mac, not the new arm64 hotness. Change-Id: Id0eb192da37dcc83646bffa9137c96b7749b179f
This commit is contained in:
Tim Burke
parent
1e90ffb844
commit
cfa07823c1
9
.dockerignore
Normal file
9
.dockerignore
Normal file
@ -0,0 +1,9 @@
|
||||
.git
|
||||
.tox
|
||||
test
|
||||
**/__pycache__
|
||||
*.egg-info
|
||||
*.so
|
||||
*.whl
|
||||
build
|
||||
dist
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@ -20,6 +20,7 @@ var/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
*.whl
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
|
||||
49
Dockerfile
Normal file
49
Dockerfile
Normal file
@ -0,0 +1,49 @@
|
||||
# manylinux2010 has oldest build chain that can still build modern ISA-L
|
||||
# 2021-02-06-3d322a5 is newest tag that still had 2.7 support
|
||||
|
||||
FROM quay.io/pypa/manylinux2010_x86_64:2021-02-06-3d322a5
|
||||
MAINTAINER OpenStack Swift
|
||||
|
||||
# can also take branch names, e.g. "master"
|
||||
ARG LIBERASURECODE_TAG=1.6.4
|
||||
ARG ISAL_TAG=v2.31.0
|
||||
|
||||
ARG SO_SUFFIX=-pyeclib
|
||||
ENV SO_SUFFIX=${SO_SUFFIX}
|
||||
ENV UID=1000
|
||||
# Alternatively, try cp27-cp27m, cp27-cp27mu
|
||||
ENV PYTHON_VERSION=cp35-cp35m
|
||||
|
||||
RUN mkdir /opt/src /output
|
||||
RUN yum install -y zlib-devel
|
||||
# Update auditwheel so it can improve our tag to manylinux1 automatically
|
||||
# Not *too far*, though, since we've got the old base image
|
||||
RUN /opt/_internal/tools/bin/pip install -U 'auditwheel<5.2'
|
||||
|
||||
# Server includes `Content-Encoding: x-gzip`, so ADD unwraps it
|
||||
ADD https://www.nasm.us/pub/nasm/releasebuilds/2.16.01/nasm-2.16.01.tar.gz /opt/src/nasm.tar
|
||||
RUN tar -C /opt/src -x -f /opt/src/nasm.tar
|
||||
RUN cd /opt/src/nasm-* && \
|
||||
./autogen.sh && \
|
||||
./configure --prefix=/usr && \
|
||||
make nasm && \
|
||||
install -c nasm /usr/bin/nasm
|
||||
|
||||
ADD https://github.com/intel/isa-l/archive/${ISAL_TAG}.tar.gz /opt/src/isa-l.tar.gz
|
||||
RUN tar -C /opt/src -x -f /opt/src/isa-l.tar.gz -z
|
||||
RUN cd /opt/src/isa-l-* && \
|
||||
./autogen.sh && \
|
||||
./configure --prefix=/usr && \
|
||||
make && \
|
||||
make install
|
||||
|
||||
ADD https://github.com/openstack/liberasurecode/archive/${LIBERASURECODE_TAG}.tar.gz /opt/src/liberasurecode.tar.gz
|
||||
RUN tar -C /opt/src -x -f /opt/src/liberasurecode.tar.gz -z
|
||||
RUN cd /opt/src/liberasurecode*/ && \
|
||||
./autogen.sh && \
|
||||
CFLAGS="-DLIBERASURECODE_SO_SUFFIX='"'"'"${SO_SUFFIX}"'"'"'" ./configure --prefix=/usr && \
|
||||
make && \
|
||||
make install
|
||||
|
||||
COPY . /opt/src/pyeclib/
|
||||
ENTRYPOINT ["/bin/sh", "-c", "/opt/python/${PYTHON_VERSION}/bin/python /opt/src/pyeclib/pack_wheel.py /opt/src/pyeclib/ --repair --so-suffix=${SO_SUFFIX} --wheel-dir=/output"]
|
||||
336
pack_wheel.py
Normal file
336
pack_wheel.py
Normal file
@ -0,0 +1,336 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Build a pyeclib wheel that contains precompiled liberasurecode libraries.
|
||||
|
||||
The goal is to build manylinux, abi3 wheels that are actually useful.
|
||||
|
||||
- ``manylinux`` ensures installability on a variety of distributions,
|
||||
almost regardless of libc version.
|
||||
- ``abi3`` ensures compatibility across a variety of python minor versions.
|
||||
- "Actually useful" means you can not only import pyeclib, but use it to
|
||||
perform encoding/decoding.
|
||||
- Where possible, we want to bundle in ISA-L support, too.
|
||||
|
||||
You might expect ``auditwheel repair`` to be able to do this for us. However,
|
||||
that's primarily designed around dynamic *linking* -- and while that's
|
||||
necessary, it's not sufficient; liberasurecode makes extensive use of dynamic
|
||||
*loading* as well, and so the dynamically loaded modules need to be included.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import errno
|
||||
import functools
|
||||
import hashlib
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import zipfile
|
||||
|
||||
|
||||
ENV_KEY = ('DYLD_LIBRARY_PATH' if sys.platform == 'darwin'
|
||||
else 'LD_LIBRARY_PATH')
|
||||
if ENV_KEY in os.environ:
|
||||
os.environ[ENV_KEY] += ':/usr/lib:/usr/local/lib'
|
||||
else:
|
||||
os.environ[ENV_KEY] = '/usr/lib:/usr/local/lib'
|
||||
|
||||
|
||||
def locate_library(name, missing_ok=False):
|
||||
"""
|
||||
Find a library.
|
||||
|
||||
:param name: The name of the library to find, not including the
|
||||
leading "lib".
|
||||
:param missing_ok: If true, return ``None`` when the library cannot be
|
||||
located.
|
||||
:raises RuntimeError: If the library cannot be found and ``missing_ok``
|
||||
is ``False``.
|
||||
:returns: The full path to the library.
|
||||
"""
|
||||
expr = r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name)
|
||||
cmd = ['ld', '-t']
|
||||
if sys.platform == 'darwin':
|
||||
cmd.extend(['-arch', platform.machine()])
|
||||
libpath = os.environ.get(ENV_KEY)
|
||||
if libpath:
|
||||
for d in libpath.split(':'):
|
||||
cmd.extend(['-L', d.rstrip('/')])
|
||||
cmd.extend(['-o', os.devnull, '-l%s' % name])
|
||||
try:
|
||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
universal_newlines=True)
|
||||
out, _ = p.communicate()
|
||||
except subprocess.CalledProcessError:
|
||||
pass
|
||||
else:
|
||||
if hasattr(os, 'fsdecode'):
|
||||
out = os.fsdecode(out)
|
||||
res = re.search(expr, out)
|
||||
if res:
|
||||
return os.path.realpath(res.group(0))
|
||||
|
||||
if missing_ok:
|
||||
return None
|
||||
|
||||
raise RuntimeError('Failed to locate %s (checked %s)' % (name, libpath))
|
||||
|
||||
|
||||
def build_wheel(src_dir):
|
||||
"""
|
||||
Build the base wheel, returning the path to the wheel.
|
||||
|
||||
Caller is responsible for cleaning up the tempdir.
|
||||
"""
|
||||
tmp = tempfile.mkdtemp()
|
||||
try:
|
||||
subprocess.check_call([
|
||||
sys.executable, 'setup.py',
|
||||
'bdist_wheel', '-d', tmp, '--py-limited-api=cp35',
|
||||
], cwd=src_dir)
|
||||
files = os.listdir(tmp)
|
||||
assert len(files) == 1, files
|
||||
return os.path.join(tmp, files[0])
|
||||
except Exception:
|
||||
shutil.rmtree(tmp)
|
||||
raise
|
||||
|
||||
|
||||
def repack_wheel(whl, so_suffix, out_whl=None):
|
||||
"""
|
||||
Repack a wheel to bundle in liberasurecode libraries.
|
||||
|
||||
This unpacks the wheel, copies all the supporting libraries to the
|
||||
unpacked wheel, adjusts rpath etc for the libraries, rebuilds the
|
||||
dist-info to include the libraries, and rebuilds the wheel.
|
||||
"""
|
||||
if out_whl is None:
|
||||
out_whl = whl
|
||||
|
||||
tmp = tempfile.mkdtemp()
|
||||
try:
|
||||
# unpack wheel
|
||||
zf = zipfile.ZipFile(whl, 'r')
|
||||
zf.extractall(tmp)
|
||||
|
||||
relocate_libs(tmp, so_suffix)
|
||||
rebuild_dist_info_record(tmp)
|
||||
build_zip(tmp, out_whl)
|
||||
finally:
|
||||
shutil.rmtree(tmp)
|
||||
|
||||
|
||||
def relocate_libs(tmp, so_suffix):
|
||||
"""
|
||||
Bundle libraries into a unpacked-wheel tree.
|
||||
|
||||
:param tmp: the temp dir containing the tree for the unzipped wheel
|
||||
:param so_suffix: the LIBERASURECODE_SO_SUFFIX used to build
|
||||
liberasurecode; this should be used to avoid
|
||||
interfering with system libraries
|
||||
"""
|
||||
lib_dir = 'pyeclib.libs'
|
||||
all_libs = [os.path.join(tmp, lib)
|
||||
for lib in os.listdir(tmp) if '.so' in lib]
|
||||
for lib in all_libs: # NB: pypy builds may create multiple .so's
|
||||
update_rpath(lib, '/' + lib_dir)
|
||||
|
||||
inject = functools.partial(
|
||||
inject_lib,
|
||||
tmp,
|
||||
so_suffix=so_suffix,
|
||||
lib_dir=lib_dir,
|
||||
all_libs=all_libs,
|
||||
)
|
||||
|
||||
# Be sure to move liberasurecode first, so we can fix its links to
|
||||
# the others
|
||||
relocated_libec = inject(locate_library('erasurecode'))
|
||||
# Since liberasurecode links against other included libraries,
|
||||
# need to update rpath
|
||||
update_rpath(relocated_libec)
|
||||
|
||||
# These guys all stand on their own, so don't need the rpath update
|
||||
inject(locate_library('nullcode'))
|
||||
inject(locate_library('Xorcode'))
|
||||
inject(locate_library('erasurecode_rs_vand'))
|
||||
|
||||
# Nobody actually links against this, but we want it anyway if available
|
||||
isal = locate_library('isal', missing_ok=True)
|
||||
if isal:
|
||||
inject(isal)
|
||||
|
||||
|
||||
def update_rpath(lib, rpath_suffix=''):
|
||||
if sys.platform == 'darwin':
|
||||
subprocess.check_call([
|
||||
'install_name_tool',
|
||||
'-add_rpath', '@loader_path' + rpath_suffix,
|
||||
lib,
|
||||
])
|
||||
else:
|
||||
subprocess.check_call([
|
||||
'patchelf', '--set-rpath', '$ORIGIN' + rpath_suffix, lib])
|
||||
|
||||
|
||||
def inject_lib(
|
||||
whl_dir,
|
||||
src_lib,
|
||||
so_suffix='-pyeclib',
|
||||
lib_dir='pyeclib.libs',
|
||||
all_libs=None,
|
||||
):
|
||||
try:
|
||||
os.mkdir(os.path.join(whl_dir, lib_dir))
|
||||
except OSError as e:
|
||||
if e.errno != errno.EEXIST:
|
||||
raise
|
||||
|
||||
if sys.platform == 'darwin':
|
||||
old_lib = src_lib
|
||||
name = os.path.basename(old_lib).split('.', 1)[0]
|
||||
new_lib = name + so_suffix + '.dylib'
|
||||
else:
|
||||
name, _, version = os.path.basename(src_lib).partition('.so')
|
||||
major = '.'.join(version.split('.', 2)[:2])
|
||||
old_lib = name + '.so' + major
|
||||
new_lib = name + so_suffix + '.so' + major
|
||||
|
||||
print('Injecting ' + new_lib)
|
||||
relocated = os.path.join(whl_dir, lib_dir, new_lib)
|
||||
shutil.copy2(src_lib, relocated)
|
||||
if sys.platform == 'darwin':
|
||||
subprocess.check_call([
|
||||
'install_name_tool', '-id', new_lib, relocated])
|
||||
else:
|
||||
subprocess.check_call(['patchelf', '--set-soname', new_lib, relocated])
|
||||
|
||||
if all_libs:
|
||||
# Fix linkage in the libs already moved -- this is mainly an issue for
|
||||
# liberasurecode.so. Jerasure *would* need it for GF-Complete, but it
|
||||
# seems unlikely we'd be able to include those any time soon
|
||||
if sys.platform == 'darwin':
|
||||
for lib in all_libs:
|
||||
subprocess.check_call([
|
||||
'install_name_tool',
|
||||
'-change', old_lib,
|
||||
'@rpath/' + new_lib,
|
||||
lib,
|
||||
])
|
||||
else:
|
||||
subprocess.check_call([
|
||||
'patchelf', '--replace-needed', old_lib, new_lib] + all_libs)
|
||||
all_libs.append(relocated)
|
||||
|
||||
return relocated
|
||||
|
||||
|
||||
def rebuild_dist_info_record(tmp):
|
||||
"""
|
||||
Update the dist-info RECORD information.
|
||||
|
||||
There are likely new files, and pre-existing files may have changed;
|
||||
rebuild the whole thing.
|
||||
|
||||
See https://packaging.python.org/en/latest/specifications/
|
||||
recording-installed-packages/#the-record-file for more info.
|
||||
"""
|
||||
tmp = tmp.rstrip('/') + '/'
|
||||
dist_info_dir = [d for d in os.listdir(tmp) if d.endswith('.dist-info')]
|
||||
assert len(dist_info_dir) == 1, dist_info_dir
|
||||
record_file = os.path.join(tmp, dist_info_dir[0], 'RECORD')
|
||||
with open(record_file, 'w') as fp:
|
||||
for dir_path, _, files in os.walk(tmp):
|
||||
for file in files:
|
||||
file = os.path.join(dir_path, file)
|
||||
if file == record_file:
|
||||
fp.write(file[len(tmp):] + ',,\n')
|
||||
continue
|
||||
hsh, sz = sha256(file)
|
||||
fp.write('%s,sha256=%s,%d\n' % (file[len(tmp):], hsh, sz))
|
||||
|
||||
|
||||
def sha256(file):
|
||||
hasher = hashlib.sha256()
|
||||
sz = 0
|
||||
with open(file, 'rb') as fp:
|
||||
for chunk in iter(lambda: fp.read(128 * 1024), b''):
|
||||
hasher.update(chunk)
|
||||
sz += len(chunk)
|
||||
hsh = base64.urlsafe_b64encode(hasher.digest())
|
||||
return hsh.decode('ascii').strip('='), sz
|
||||
|
||||
|
||||
def build_zip(tmp, out_file):
|
||||
"""
|
||||
Zip up all files in a tree, with archive names relative to the root.
|
||||
"""
|
||||
tmp = tmp.rstrip('/') + '/'
|
||||
with zipfile.ZipFile(out_file, 'w') as zf:
|
||||
for dir_path, _, files in os.walk(tmp):
|
||||
for file in files:
|
||||
file = os.path.join(dir_path, file)
|
||||
zf.write(file, file[len(tmp):])
|
||||
|
||||
|
||||
def repair_wheel(whl):
|
||||
"""
|
||||
Run ``auditwheel repair`` to ensure appropriate platform tags.
|
||||
"""
|
||||
if sys.platform == 'darwin':
|
||||
return whl # auditwheel only works on linux
|
||||
whl_dir = os.path.dirname(whl)
|
||||
subprocess.check_call(['auditwheel', 'repair', whl, '-w', whl_dir])
|
||||
wheels = [f for f in os.listdir(whl_dir) if f != os.path.basename(whl)]
|
||||
assert len(wheels) == 1, wheels
|
||||
return os.path.join(whl_dir, wheels[0])
|
||||
|
||||
|
||||
def fix_ownership(whl):
|
||||
uid = int(os.environ.get('UID', '1000'))
|
||||
gid = int(os.environ.get('GID', uid))
|
||||
os.chown(whl, uid, gid)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('src_dir')
|
||||
parser.add_argument('-w', '--wheel-dir', default='.')
|
||||
parser.add_argument('-s', '--so-suffix', default='-pyeclib')
|
||||
parser.add_argument('-r', '--repair', action='store_true')
|
||||
args = parser.parse_args()
|
||||
whl = build_wheel(args.src_dir)
|
||||
whl_dir = os.path.dirname(whl)
|
||||
try:
|
||||
repack_wheel(whl, args.so_suffix)
|
||||
if args.repair:
|
||||
whl = repair_wheel(whl)
|
||||
output_whl = os.path.join(
|
||||
args.wheel_dir, os.path.basename(whl))
|
||||
shutil.move(whl, output_whl)
|
||||
if os.geteuid() == 0:
|
||||
# high likelihood of running in a docker container or something
|
||||
fix_ownership(output_whl)
|
||||
finally:
|
||||
shutil.rmtree(whl_dir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user