build-wheels: README file + minor script fixes

Add a textfile describing the wheels cfg file format.

Allow tarballs that pack their contents at top level.

Story: 2010294
Task: 46279

Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
Change-Id: Ia28ca95d1be1a81e8481709139fc440f336de7f7

build-tools/build-wheels/centos/dev-wheels-py2.cfg

@@ -6,6 +6,8 @@
 #
 # If fix_setup must be called, add |fix_setup at the end of the line
 #
+# See doc/wheels-cfg.md for more info.
+#
 amqplib-1.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2
 lz4-0.9.0-cp27-cp27mu-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0
 panko-5.0.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a9/89/d666e0889d869e41c9b7f87a0a34858b2520782b82e025da84c98e0db8f6/panko-5.0.0.tar.gz|panko-5.0.0

build-tools/build-wheels/centos/dev-wheels.cfg

@@ -6,6 +6,8 @@
 #
 # If fix_setup must be called, add |fix_setup at the end of the line
 #
+# See doc/wheels-cfg.md for more info.
+#
 amqplib-1.0.2-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2
 croniter-0.3.29-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/a9/c9/11182a2507798c661b04a7914739ea8ca73a738e6869a23742029f51bc1a/croniter-0.3.29-py2.py3-none-any.whl
 google_api_python_client-1.7.7-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/d7/47/940908e52487440f61fb93ad55cbbe3a28235d3bb143b26affb17b37dd28/google_api_python_client-1.7.7-py2.py3-none-any.whl

build-tools/build-wheels/centos/stable-wheels-py2.cfg

@@ -6,6 +6,8 @@
 #
 # If fix_setup must be called, add |fix_setup at the end of the line
 #
+# See doc/wheels-cfg.md for more info.
+#
 abclient-0.2.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/49/eb/091b02c1e36d68927adfb746706e2c80f7e7bfb3f16e3cbcfec2632118ab/abclient-0.2.3.tar.gz|abclient-0.2.3
 alembic-1.1.0-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/9a/0f/a5e8997d58882da8ecd288360dddf133a83145de6480216774923b393422/alembic-1.1.0.tar.gz|alembic-1.1.0
 amqplib-1.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2

build-tools/build-wheels/centos/stable-wheels.cfg

@@ -6,6 +6,8 @@
 #
 # If fix_setup must be called, add |fix_setup at the end of the line
 #
+# See doc/wheels-cfg.md for more info.
+#
 abclient-0.2.3-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/49/eb/091b02c1e36d68927adfb746706e2c80f7e7bfb3f16e3cbcfec2632118ab/abclient-0.2.3.tar.gz|abclient-0.2.3
 alembic-1.4.2-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/60/1e/cabc75a189de0fbb2841d0975243e59bde8b7822bacbb95008ac6fe9ad47/alembic-1.4.2.tar.gz|alembic-1.4.2
 amqplib-1.0.2-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2

build-tools/build-wheels/debian/stable-wheels-py2.cfg

@@ -6,6 +6,8 @@
 #
 # If fix_setup must be called, add |fix_setup at the end of the line
 #
+# See doc/wheels-cfg.md for more info.
+#
 abclient-0.2.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/49/eb/091b02c1e36d68927adfb746706e2c80f7e7bfb3f16e3cbcfec2632118ab/abclient-0.2.3.tar.gz|abclient-0.2.3
 alembic-1.1.0-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/9a/0f/a5e8997d58882da8ecd288360dddf133a83145de6480216774923b393422/alembic-1.1.0.tar.gz|alembic-1.1.0
 amqplib-1.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2

build-tools/build-wheels/debian/stable-wheels.cfg

@@ -6,6 +6,8 @@
 #
 # If fix_setup must be called, add |fix_setup at the end of the line
 #
+# See doc/wheels-cfg.md for more info.
+#
 abclient-0.2.3-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/49/eb/091b02c1e36d68927adfb746706e2c80f7e7bfb3f16e3cbcfec2632118ab/abclient-0.2.3.tar.gz|abclient-0.2.3
 alembic-1.4.2-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/60/1e/cabc75a189de0fbb2841d0975243e59bde8b7822bacbb95008ac6fe9ad47/alembic-1.4.2.tar.gz|alembic-1.4.2
 amqplib-1.0.2-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2

build-tools/build-wheels/doc/wheels-cfg.md

@@ -0,0 +1,65 @@
+# Syntax of wheels config files
+
+The files {debian,centos}/{stable,dev}-wheels.cfg list the 3rd-party wheels
+(ie compiled python modules) to be included in the wheels tarball. Wheels are
+listed one per line, each with the following "|"-separated fields.
+
+The first field is the wheel file name and is often python version & ABI
+specific, eg: `lz4-0.9.0-cp39-cp39-linux_x86_64.whl`.
+
+The second field is `git`, `tar`, `zip`, or `pypi` and determines how to build
+or download the wheel
+
+The third field is the URL of wheel source of the wheel itself
+
+The optional last field may be set to `fix_setup`, which will update
+older/legacy module sources to use setuptools.setup, which is necessary for
+building the module into a wheel. See script
+`docker-common/docker-build-wheel.sh` for details.
+
+The exact number of fields depend on the wheel source:
+
+* git: `wheelname|git|repo-url|basedir|branch|fix_setup`
+
+  Remote git repo URL + branch
+
+  Example:
+
+  ```
+  lz4-0.9.0-cp39-cp39-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0
+  ```
+
+* tar: `wheelname|tar|url|basedir|fix_setup`
+
+  Source tarball URL. Basedir must be the subdirectory containing the package
+  source code within the tarball, and typically equals the basename of the
+  tarfile and/or module name & version. If tarfile contains source files at
+  top level, this parameter must be empty or "."
+
+  Example:
+
+  ```
+  abclient-0.2.3-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/49/eb/091b02c1e36d68927adfb746706e2c80f7e7bfb3f16e3cbcfec2632118ab/abclient-0.2.3.tar.gz|abclient-0.2.3i
+  ```
+
+* zip: `wheelname|zip|url|basedir|fix_setup`
+
+  Same as `tar`, but in .zip format
+
+  Example:
+
+  ```
+  networkx-2.2-py2.py3-none-any.whl|zip|https://files.pythonhosted.org/packages/f3/f4/7e20ef40b118478191cec0b58c3192f822cace858c19505c7670961b76b2/networkx-2.2.zip|networkx-2.2
+  ```
+
+* pypi: `wheelname|pypi|url`
+
+  URL of a pre-built wheel for direct download. This type of download avoids
+  the build step and reduces overall build times.
+
+  Example:
+
+  ```
+  bottle-0.12.18-py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/e9/39/2bf3a1fd963e749cdbe5036a184eda8c37d8af25d1297d94b8b7aeec17c4/bottle-0.12.18-py3-none-any.whl
+  ```
+

build-tools/build-wheels/docker-common/docker-build-wheel.sh

@@ -218,11 +218,11 @@ function from_tar {
 
         tarball=$(basename $wgetsrc)
         if [[ $tarball =~ gz$ ]]; then
-            taropts="xzf"
+            taropts="-xzf"
         elif [[ $tarball =~ bz2$ ]]; then
-            taropts="xjf"
+            taropts="-xjf"
         else
-            taropts="xf"
+            taropts="-xf"
         fi
 
         with_retries ${MAX_ATTEMPTS} wget $wgetsrc
@@ -231,7 +231,14 @@ function from_tar {
             continue
         fi
 
-        tar $taropts $(basename $wgetsrc)
+        local -a tar_base_opts
+        if [[ -z "$basedir" || "$basedir" == "." ]] ; then
+            basedir="$(echo "$tarball" | sed -r -e 's#[.]tar([.].*)?$##')"
+            mkdir -p "$basedir"
+            tar_basedir_opts+=("-C" "$basedir")
+        fi
+
+        tar "${tar_basedir_opts[@]}" $taropts $(basename $wgetsrc)
         if [ $? -ne 0 ]; then
             echo "Failure running: tar $taropts $(basename $wgetsrc)"
             echo $wheelname >> $FAILED_LOG