From 9baff8aa884a4169e5a0bd2ae2e8463ee1de7ee2 Mon Sep 17 00:00:00 2001 From: "M. Vefa Bicakci" Date: Wed, 16 Feb 2022 12:08:59 -0500 Subject: [PATCH] Introduce libbnxt_re version 220.0.5.0 This commit introduces "libbnxt_re" version 220.0.5.0 to StarlingX to support Broadcom NetXtreme-E network adapters' Infiniband/RDMA functionality. "libbnxt_re" is an Infiniband verbs library. The source RPM file is used for packaging purposes, and the following changes are made to source RPM as part of the packaging for StarlingX. (This is a summary; for further information, please see the patch files.) - The spec file is adapted to StarlingX by modifying its Release and BuildRoot fields. - For compatibility with StarlingX's build scripts, the whitespace between the field names and the colon characters (':') are removed. - A BuildRequires is added for kernel{?bt_ext}-headers >= 5.10 to ensure that libbnxt_re build does not fail due to a conflict between rdma-core and kernel-headers-4.18, the latter of which is pre-installed and stays installed in the mock/chroot file system until the kernel packages are built by the build system. (This build failure is a transient issue impacting efficiency, as the build system retries building the package until the correct kernel headers are installed.) - The configure script is regenerated to ensure that the newer version of rdma-core is supported by the configure script, which hard-codes ABI version v22, whereas StarlingX ships with rdma-core ABI version v37. This is done by calling the autogen.sh script in %build. - The original source RPM file replaces pre-existing libbnxt_re library file symbolic link in /usr/lib64/libibverbs via post-installation hooks (i.e., %post). This behaviour is modified to instead directly include the symbolic link in the package, since StarlingX's version of rdma-core does not include the libbnxt_re library. - The configure.ac file is patched to use a compatibility version of the "stdatomic.h" header provided by libbnxt_re, as the StarlingX/CentOS 7 toolchain does not have this header file. Testing: - An ISO image is successfully built (monolithically) with this commit. - This commit has not yet been directly tested due to a lack of hardware; however, we have confirmed that the library is successfully dynamically loaded by applications making use of the libibverbs library by packaging the Linux RDMA community's perftest tools for StarlingX and then starting the ib_send_bw performance test tool on a physical server with Mellanox hardware. This was followed by confirming that the /proc//maps file for the ib_send_bw process contains the new libbnxt_re library. The latter indicates that the library is successfully loaded. Story: 2009915 Task: 44759 Depends-On: https://review.opendev.org/c/starlingx/kernel/+/833464 Depends-On: https://review.opendev.org/c/starlingx/tools/+/833462 Change-Id: I3ffd0a0cbaa076802c8140d8fe67cbe56e6b3a90 Signed-off-by: M. Vefa Bicakci --- centos_extra_downloads.lst | 1 + centos_iso_image.inc | 1 + centos_pkg_dirs | 1 + centos_pkg_dirs_rt | 1 + .../libbnxt_re/centos/build_srpm.data | 5 + ...1-libbnxt.spec-StarlingX-adaptations.patch | 179 ++++++++++++++++++ .../centos/meta_patches/PATCH_ORDER | 1 + ...0001-Set-up-stdatomic.h-if-necessary.patch | 49 +++++ .../broadcom/libbnxt_re/centos/srpm_path | 1 + 9 files changed, 239 insertions(+) create mode 100644 userspace/broadcom/libbnxt_re/centos/build_srpm.data create mode 100644 userspace/broadcom/libbnxt_re/centos/meta_patches/0001-libbnxt.spec-StarlingX-adaptations.patch create mode 100644 userspace/broadcom/libbnxt_re/centos/meta_patches/PATCH_ORDER create mode 100644 userspace/broadcom/libbnxt_re/centos/patches/0001-Set-up-stdatomic.h-if-necessary.patch create mode 100644 userspace/broadcom/libbnxt_re/centos/srpm_path diff --git a/centos_extra_downloads.lst b/centos_extra_downloads.lst index 65024730..51b17f59 100644 --- a/centos_extra_downloads.lst +++ b/centos_extra_downloads.lst @@ -5,3 +5,4 @@ rdma-core-55mlnx37-1.55103.src.rpm opae-intel-fpga-driver-2.0.1-10.src.rpm ice_comms-1.3.31.0.zip netxtreme-bnxt_en-1.10.2-220.0.13.0.tar.gz +libbnxt_re-220.0.5.0-rhel7u9.src.rpm diff --git a/centos_iso_image.inc b/centos_iso_image.inc index 8f4f92ae..432dc9f0 100644 --- a/centos_iso_image.inc +++ b/centos_iso_image.inc @@ -64,3 +64,4 @@ mstflint # Broadcom device driver kmod-bnxt_en kmod-bnxt_en-rt +libbnxt_re diff --git a/centos_pkg_dirs b/centos_pkg_dirs index db6c85cc..5422010c 100644 --- a/centos_pkg_dirs +++ b/centos_pkg_dirs @@ -7,6 +7,7 @@ kernel-modules/intel-igb_uio kernel-modules/qat17 kernel-modules/intel-opae-fpga kernel-modules/mlnx-ofa_kernel +userspace/broadcom/libbnxt_re userspace/mellanox/rdma-core userspace/mellanox/mlnx-tools userspace/mellanox/mstflint diff --git a/centos_pkg_dirs_rt b/centos_pkg_dirs_rt index b83d93db..d682be57 100644 --- a/centos_pkg_dirs_rt +++ b/centos_pkg_dirs_rt @@ -7,6 +7,7 @@ kernel-modules/intel-ice kernel-modules/intel-igb_uio kernel-modules/qat17 kernel-modules/intel-opae-fpga +userspace/broadcom/libbnxt_re userspace/mellanox/rdma-core userspace/mellanox/mlnx-tools userspace/mellanox/mstflint diff --git a/userspace/broadcom/libbnxt_re/centos/build_srpm.data b/userspace/broadcom/libbnxt_re/centos/build_srpm.data new file mode 100644 index 00000000..5066b7db --- /dev/null +++ b/userspace/broadcom/libbnxt_re/centos/build_srpm.data @@ -0,0 +1,5 @@ +COPY_LIST=" \ + $PKG_BASE/centos/patches/* \ + " + +TIS_PATCH_VER=PKG_GITREVCOUNT diff --git a/userspace/broadcom/libbnxt_re/centos/meta_patches/0001-libbnxt.spec-StarlingX-adaptations.patch b/userspace/broadcom/libbnxt_re/centos/meta_patches/0001-libbnxt.spec-StarlingX-adaptations.patch new file mode 100644 index 00000000..1cdc8435 --- /dev/null +++ b/userspace/broadcom/libbnxt_re/centos/meta_patches/0001-libbnxt.spec-StarlingX-adaptations.patch @@ -0,0 +1,179 @@ +From 78be296d1e11aa86c684a5a263083e3cfd546cdc Mon Sep 17 00:00:00 2001 +From: "M. Vefa Bicakci" +Date: Wed, 16 Feb 2022 15:12:25 -0500 +Subject: [PATCH] libbnxt.spec: StarlingX adaptations + +This patch adapts libbnxt.spec to StarlingX as follows: + +* The Release field is modified and BuildRoot field is removed for + StarlingX. + +* StarlingX's build scripts cannot successfully parse the spec file when + there are space characters between the spec file's field names (such + as "Version") and the colons (":") following the field names. Hence, + this patch removes all whitespace between field names and colon + characters. + +* The source RPM provided by Broadcom has a "configure" script + pre-generated for Red Hat Enterprise Linux 7.9, which hardcodes the + variable RDMA_CORE_VERSION to v22; however, StarlingX has a newer + version of the rdma-core library (v37 as of this writing) included, + which requires StarlingX build to regenerate the configure script from + configure.ac. This requires various autotools packages as well as + making the autogen.sh script executable and calling the latter script + prior to calling configure. + +* A build-time dependency on the kernel headers >= 5.10 is added to + ensure that the expected kernel{,-rt}-headers package is installed + into the build chroot/mock file system prior to attempting to build + libbnxt_re. + + If the aforementioned BuildRequires is not present, then the + pre-existing "kernel-headers-4.18.0-147.3.1.el8_1.x86_64" package in + the chroot/mock file system is found satisfactory by rpmbuild, which + triggers the following compilation errors, caused by a conflict + between the header files in the rdma-core-devel package and the + aforementioned kernel{,-rt}-headers package: + + In file included from ./src/rc-compat/v35/cmd_ioctl.h:40:0, + from ./src/rc-compat/v35/driver.h:42, + from src/main.h:55, + from src/verbs.c:56: + /usr/include/rdma/rdma_user_ioctl_cmds.h:87:6: error: nested redefinition of 'enum rdma_driver_id' + enum rdma_driver_id { + ^ + /usr/include/rdma/rdma_user_ioctl_cmds.h:87:6: error: redeclaration of 'enum rdma_driver_id' + In file included from /usr/include/infiniband/verbs_api.h:66:0, + from /usr/include/infiniband/verbs.h:48, + from ./src/rc-compat/v35/driver.h:40, + from src/main.h:55, + from src/verbs.c:56: + /usr/include/infiniband/ib_user_ioctl_verbs.h:232:6: note: originally defined here + enum rdma_driver_id { + ^ + +* Broadcom's libbnxt RPMs replace pre-existing libbnxt_re libraries from + /usr/lib64/libibverbs at package installation time and restore the + older versions at package removal time via post-install and + post-uninstall directives. rdma-core in StarlingX does not package + libbnxt_re, which allows the libbnxt_re package to provide this + library. Hence, this patch instead modifies the spec file so that the + library symbolic link in /usr/lib64/libibverbs is placed at package + build time instead of package installation time. + +* This patch also modifies the spec file to apply a patch that allows + the libbnxt_re build to succeed with StarlingX's older CentOS 7 + toolchain which does not provide the "stdatomic.h" header. + +* Finally, a SuSe-related section of the spec file is trimmed. + +Signed-off-by: M. Vefa Bicakci +--- + SPECS/libbnxt.spec | 61 ++++++++++++++++++++++++++-------------------- + 1 file changed, 34 insertions(+), 27 deletions(-) + +diff --git a/SPECS/libbnxt.spec b/SPECS/libbnxt.spec +index 173229da92a5..992bda9e18ff 100644 +--- a/SPECS/libbnxt.spec ++++ b/SPECS/libbnxt.spec +@@ -1,23 +1,28 @@ +-Name : libbnxt_re +-Version : 220.0.5.0 +-Release : rhel7u9 +-Summary : Userspace Library for Broadcom ROCE Device. +-Group : System Environment/Libraries +-License : GPL/BSD +-Vendor : Broadcom Limited +-URL : http://www.broadcom.com +-Source : libbnxt_re-220.0.5.0.tar.gz +-BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) ++%if "%{_tis_build_type}" == "rt" ++%define bt_ext -rt ++%else ++%undefine bt_ext ++%endif ++ ++Name: libbnxt_re ++Version: 220.0.5.0 ++Release: rhel7u9%{?_tis_dist}.%{tis_patch_ver} ++Summary: Userspace Library for Broadcom ROCE Device. ++Group: System Environment/Libraries ++License: GPL/BSD ++Vendor: Broadcom Limited ++URL: http://www.broadcom.com ++Source: libbnxt_re-220.0.5.0.tar.gz + BuildRequires: rdma-core-devel ++BuildRequires: automake autoconf libtool ++BuildRequires: kernel%{?bt_ext}-headers >= 5.10 ++ ++Patch0001: 0001-Set-up-stdatomic.h-if-necessary.patch + + %description + libbnxt_re provides a device-specific userspace driver for Broadcom Netxtreme RoCE Adapters + for use with the libibverbs library. + +-%if %{defined suse_version} +-%debug_package +-%endif +- + %package devel + Summary: Development files for the libbnxt_re driver + Group: System Environment/Libraries +@@ -29,8 +34,11 @@ application, which may be useful for debugging. + + %prep + %setup -q -n %{name}-%{version} ++%patch0001 -p1 + + %build ++chmod a+x autogen.sh ++./autogen.sh + %configure + make %{?_smp_mflags} + +@@ -40,23 +48,21 @@ rm -rf $RPM_BUILD_ROOT + # remove unpackaged files from the buildroot + rm -f $RPM_BUILD_ROOT%{_libdir}/*.la + ++# Now, the following symbolic link exists: ++# /usr/lib64/libbnxt_re.so -> libbnxt_re-rdmav34.so ++# ++# We need to create the following symbolic link as well: ++# /usr/lib64/libibverbs/libbnxt_re-rdmav34.so -> ../libbnxt_re-rdmav34.so ++# ++# To avoid hard-coding the version (v34), read the link. ++libname="$(readlink $RPM_BUILD_ROOT%{_libdir}/libbnxt_re.so)" ++mkdir -p $RPM_BUILD_ROOT%{_libdir}/libibverbs/ ++ln -s "../${libname}" $RPM_BUILD_ROOT%{_libdir}/libibverbs/"${libname}" ++ + %post +-# If libbnxt_re is inboxed +-if [ -f "%{_libdir}/libibverbs/libbnxt_re-rdmav"*".so" ];then +- #to get full name and path +- libbnxtre=$(ls "%{_libdir}/libibverbs/libbnxt_re-rdmav"*".so") +- libbnxtre_mod=$(echo $libbnxtre | sed "s/libbnxt_re-/old.libbnxt_re-/") +- mv $libbnxtre $libbnxtre_mod > /dev/null 2>&1 +-fi + /sbin/ldconfig + + %postun +-# Undo the change if we had renamed the file +-if [ -f "%{_libdir}/libibverbs/old.libbnxt_re-rdmav"*".so" ];then +- libbnxtre_mod=$(ls "%{_libdir}/libibverbs/old.libbnxt_re-rdmav"*".so") +- libbnxtre=$(echo $libbnxtre_mod | sed "s/old.//") +- mv $libbnxtre_mod $libbnxtre > /dev/null 2>&1 +-fi + /sbin/ldconfig + + %clean +@@ -65,6 +71,7 @@ rm -rf $RPM_BUILD_ROOT + %files + %defattr(-,root,root,-) + %{_libdir}/libbnxt_re*.so ++%{_libdir}/libibverbs/libbnxt_re*.so + #%doc AUTHORS COPYING ChangeLog README + %config %{_sysconfdir}/libibverbs.d/bnxt_re.driver + +-- +2.29.2 + diff --git a/userspace/broadcom/libbnxt_re/centos/meta_patches/PATCH_ORDER b/userspace/broadcom/libbnxt_re/centos/meta_patches/PATCH_ORDER new file mode 100644 index 00000000..2f1f62c7 --- /dev/null +++ b/userspace/broadcom/libbnxt_re/centos/meta_patches/PATCH_ORDER @@ -0,0 +1 @@ +0001-libbnxt.spec-StarlingX-adaptations.patch diff --git a/userspace/broadcom/libbnxt_re/centos/patches/0001-Set-up-stdatomic.h-if-necessary.patch b/userspace/broadcom/libbnxt_re/centos/patches/0001-Set-up-stdatomic.h-if-necessary.patch new file mode 100644 index 00000000..fe066a99 --- /dev/null +++ b/userspace/broadcom/libbnxt_re/centos/patches/0001-Set-up-stdatomic.h-if-necessary.patch @@ -0,0 +1,49 @@ +From 05fe24097fa8287bddea3418799e32d949999267 Mon Sep 17 00:00:00 2001 +From: "M. Vefa Bicakci" +Date: Mon, 14 Feb 2022 13:13:09 -0500 +Subject: [PATCH] Set up stdatomic.h if necessary + +The toolchain used for building libraries in StarlingX (i.e., CentOS 7) +does not provide the stdatomic.h header. Thankfully, the libbnxt_re +package includes a compatibility-oriented version of this header in +rc-compat/v22. + +Hence, copy over this header from the rc-compat/v22 directory to the +current rc-compat version sub-directory, but only if required, to let +the build finish successfully. + +Signed-off-by: M. Vefa Bicakci +--- + configure.ac | 16 ++++++++++++++++ + 1 file changed, 16 insertions(+) + +diff --git a/configure.ac b/configure.ac +index 9e4e980886d4..61f2a014eb94 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -481,6 +481,22 @@ AC_HEADER_STDC + dnl Checks for typedefs, structures, and compiler characteristics. + AC_C_CONST + ++dnl Check if the host's toolchain provides stdatomic.h and use ++dnl our local copy if required. ++tmp_stdatomic=if$$.c ++echo "#include " > ${tmp_stdatomic} ++dnl Need to work around embedded newlines, hence the additional ++dnl level of indirection. ++incpath=$(echo -n __include_path__) ++if echo -- $incpath | grep -q src/rc-compat/ && \ ++ ! $CC $CPPFLAGS -E ${tmp_stdatomic} >/dev/null 2>/dev/null && \ ++ ! test -f src/rc-compat/$(echo -n rcore_version)/stdatomic.h; then ++ cp -v src/rc-compat/v22/stdatomic.h src/rc-compat/$(echo -n rcore_version)/stdatomic.h ++fi ++rm -f ${tmp_stdatomic} ++unset tmp_stdatomic ++unset incpath ++ + dnl Check for libibverbs device library extension + dummy=if$$ + cat < $dummy.c +-- +2.29.2 + diff --git a/userspace/broadcom/libbnxt_re/centos/srpm_path b/userspace/broadcom/libbnxt_re/centos/srpm_path new file mode 100644 index 00000000..6228409d --- /dev/null +++ b/userspace/broadcom/libbnxt_re/centos/srpm_path @@ -0,0 +1 @@ +repo:stx/downloads/libbnxt_re-220.0.5.0-rhel7u9.src.rpm