From a68570b5d92062ef4a8e80191c7521a09f0a9221 Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Mon, 23 Oct 2023 19:28:39 +0800 Subject: [PATCH] Add computing offloading code 1. Add computing offloading code 2. Add script.md 3. Add virsh_demo.xml Change-Id: Id9ef883e2f0eb727eb5448b9d1c47767f46b1021 Signed-off-by: Yikun Jiang --- README.md | 8 + qtfs/CMakeLists.txt | 40 + qtfs/LICENSE | 127 ++ qtfs/README.md | 116 ++ qtfs/config/qtfs/whitelist | 35 + qtfs/config/rexec/whitelist | 1 + qtfs/demo/Makefile | 12 + qtfs/demo/cfifo_r.c | 98 + qtfs/demo/cfifo_w.c | 30 + qtfs/doc/ Overall_architecture_diagram.png | Bin 0 -> 49089 bytes qtfs/doc/figures/offload-arch.png | Bin 0 -> 39249 bytes qtfs/doc/figures/qtfs-arch.png | Bin 0 -> 49089 bytes qtfs/doc/overview.md | 11 + qtfs/doc/public_sys-resources/icon-note.gif | Bin 0 -> 394 bytes qtfs/doc/qtfs共享文件系统架构及使用手册.md | 69 + qtfs/doc/容器管理面无感卸载.md | 31 + qtfs/doc/无感卸载部署指导.md | 166 ++ qtfs/include/comm.h | 322 +++ qtfs/include/conn.h | 219 +++ qtfs/include/log.h | 126 ++ qtfs/include/qtfs_check.h | 59 + qtfs/include/req.h | 562 ++++++ qtfs/include/symbol_wrapper.h | 109 ++ qtfs/ipc/Makefile | 38 + qtfs/ipc/uds_connector.c | 160 ++ qtfs/ipc/uds_event.c | 1096 +++++++++++ qtfs/ipc/uds_event.h | 81 + qtfs/ipc/uds_main.c | 758 +++++++ qtfs/ipc/uds_main.h | 160 ++ qtfs/ipc/uds_module.h | 37 + qtfs/qtfs/CMakeLists.txt | 75 + qtfs/qtfs/License | 340 ++++ qtfs/qtfs/Makefile | 27 + qtfs/qtfs/fifo.c | 221 +++ qtfs/qtfs/miss.c | 82 + qtfs/qtfs/ops.h | 43 + qtfs/qtfs/proc.c | 310 +++ qtfs/qtfs/qtfs-mod.c | 316 +++ qtfs/qtfs/qtfs-mod.h | 196 ++ qtfs/qtfs/sb.c | 1668 ++++++++++++++++ qtfs/qtfs/syscall.c | 434 +++++ qtfs/qtfs/syscall.h | 20 + qtfs/qtfs/xattr.c | 318 +++ qtfs/qtfs_common/License | 340 ++++ qtfs/qtfs_common/conn.c | 951 +++++++++ qtfs/qtfs_common/misc.c | 355 ++++ qtfs/qtfs_common/qtfs_check.c | 332 ++++ qtfs/qtfs_common/socket.c | 544 ++++++ qtfs/qtfs_common/symbol_wrapper.c | 296 +++ qtfs/qtfs_common/user_engine.c | 521 +++++ qtfs/qtfs_server/CMakeLists.txt | 69 + qtfs/qtfs_server/License | 340 ++++ qtfs/qtfs_server/Makefile | 50 + qtfs/qtfs_server/fsops.c | 1734 +++++++++++++++++ qtfs/qtfs_server/fsops.h | 25 + qtfs/qtfs_server/qtfs-server.c | 440 +++++ qtfs/qtfs_server/qtfs-server.h | 51 + qtfs/qtfs_server/qtfs_fifo_server/Cargo.toml | 9 + .../qtfs_fifo_server/src/cofifo.rs | 337 ++++ qtfs/qtfs_server/qtfs_fifo_server/src/main.rs | 72 + qtfs/qtinfo/Makefile | 23 + qtfs/qtinfo/qtinfo.c | 627 ++++++ qtfs/qtinfo/qtinfo.h | 78 + qtfs/rexec/Makefile | 26 + qtfs/rexec/README.md | 36 + qtfs/rexec/rexec.c | 773 ++++++++ qtfs/rexec/rexec.h | 170 ++ qtfs/rexec/rexec_server.c | 615 ++++++ qtfs/rexec/rexec_shim.c | 157 ++ qtfs/rexec/rexec_sock.c | 177 ++ qtfs/rexec/rexec_sock.h | 40 + qtfs/test/cgroup.go | 53 + qtfs/test/pjdfstest/README.md | 30 + qtfs/test/qtfs_test/test_fifo_block.sh | 32 + qtfs/test/qtfs_test/test_handle_open.rs | 53 + .../test/rexec_test/test_rexec_pipe_inherit.c | 23 + .../test_rexec_pipe_inherit_remote.c | 13 + qtfs/test/scmright_test/README.md | 39 + qtfs/test/scmright_test/client.c | 81 + qtfs/test/scmright_test/makefile | 15 + qtfs/test/scmright_test/scm_rights.c | 96 + qtfs/test/scmright_test/server.c | 88 + qtfs/test/scmright_test/uds.h | 9 + qtfs/utils/License | 340 ++++ qtfs/utils/qtfs_utils.h | 55 + qtfs/utils/utils.c | 202 ++ script.md | 66 + virsh_demo.xml | 110 ++ 88 files changed, 19014 insertions(+) create mode 100644 qtfs/CMakeLists.txt create mode 100644 qtfs/LICENSE create mode 100644 qtfs/README.md create mode 100644 qtfs/config/qtfs/whitelist create mode 100644 qtfs/config/rexec/whitelist create mode 100644 qtfs/demo/Makefile create mode 100644 qtfs/demo/cfifo_r.c create mode 100644 qtfs/demo/cfifo_w.c create mode 100644 qtfs/doc/ Overall_architecture_diagram.png create mode 100644 qtfs/doc/figures/offload-arch.png create mode 100644 qtfs/doc/figures/qtfs-arch.png create mode 100644 qtfs/doc/overview.md create mode 100644 qtfs/doc/public_sys-resources/icon-note.gif create mode 100644 qtfs/doc/qtfs共享文件系统架构及使用手册.md create mode 100644 qtfs/doc/容器管理面无感卸载.md create mode 100644 qtfs/doc/无感卸载部署指导.md create mode 100644 qtfs/include/comm.h create mode 100644 qtfs/include/conn.h create mode 100644 qtfs/include/log.h create mode 100644 qtfs/include/qtfs_check.h create mode 100644 qtfs/include/req.h create mode 100644 qtfs/include/symbol_wrapper.h create mode 100644 qtfs/ipc/Makefile create mode 100644 qtfs/ipc/uds_connector.c create mode 100644 qtfs/ipc/uds_event.c create mode 100644 qtfs/ipc/uds_event.h create mode 100644 qtfs/ipc/uds_main.c create mode 100644 qtfs/ipc/uds_main.h create mode 100644 qtfs/ipc/uds_module.h create mode 100644 qtfs/qtfs/CMakeLists.txt create mode 100644 qtfs/qtfs/License create mode 100644 qtfs/qtfs/Makefile create mode 100644 qtfs/qtfs/fifo.c create mode 100644 qtfs/qtfs/miss.c create mode 100644 qtfs/qtfs/ops.h create mode 100644 qtfs/qtfs/proc.c create mode 100644 qtfs/qtfs/qtfs-mod.c create mode 100644 qtfs/qtfs/qtfs-mod.h create mode 100644 qtfs/qtfs/sb.c create mode 100644 qtfs/qtfs/syscall.c create mode 100644 qtfs/qtfs/syscall.h create mode 100644 qtfs/qtfs/xattr.c create mode 100644 qtfs/qtfs_common/License create mode 100644 qtfs/qtfs_common/conn.c create mode 100644 qtfs/qtfs_common/misc.c create mode 100644 qtfs/qtfs_common/qtfs_check.c create mode 100644 qtfs/qtfs_common/socket.c create mode 100644 qtfs/qtfs_common/symbol_wrapper.c create mode 100644 qtfs/qtfs_common/user_engine.c create mode 100644 qtfs/qtfs_server/CMakeLists.txt create mode 100644 qtfs/qtfs_server/License create mode 100644 qtfs/qtfs_server/Makefile create mode 100644 qtfs/qtfs_server/fsops.c create mode 100644 qtfs/qtfs_server/fsops.h create mode 100644 qtfs/qtfs_server/qtfs-server.c create mode 100644 qtfs/qtfs_server/qtfs-server.h create mode 100644 qtfs/qtfs_server/qtfs_fifo_server/Cargo.toml create mode 100644 qtfs/qtfs_server/qtfs_fifo_server/src/cofifo.rs create mode 100644 qtfs/qtfs_server/qtfs_fifo_server/src/main.rs create mode 100644 qtfs/qtinfo/Makefile create mode 100644 qtfs/qtinfo/qtinfo.c create mode 100644 qtfs/qtinfo/qtinfo.h create mode 100644 qtfs/rexec/Makefile create mode 100644 qtfs/rexec/README.md create mode 100644 qtfs/rexec/rexec.c create mode 100644 qtfs/rexec/rexec.h create mode 100644 qtfs/rexec/rexec_server.c create mode 100644 qtfs/rexec/rexec_shim.c create mode 100644 qtfs/rexec/rexec_sock.c create mode 100644 qtfs/rexec/rexec_sock.h create mode 100644 qtfs/test/cgroup.go create mode 100644 qtfs/test/pjdfstest/README.md create mode 100644 qtfs/test/qtfs_test/test_fifo_block.sh create mode 100644 qtfs/test/qtfs_test/test_handle_open.rs create mode 100644 qtfs/test/rexec_test/test_rexec_pipe_inherit.c create mode 100644 qtfs/test/rexec_test/test_rexec_pipe_inherit_remote.c create mode 100644 qtfs/test/scmright_test/README.md create mode 100644 qtfs/test/scmright_test/client.c create mode 100644 qtfs/test/scmright_test/makefile create mode 100644 qtfs/test/scmright_test/scm_rights.c create mode 100644 qtfs/test/scmright_test/server.c create mode 100644 qtfs/test/scmright_test/uds.h create mode 100644 qtfs/utils/License create mode 100644 qtfs/utils/qtfs_utils.h create mode 100644 qtfs/utils/utils.c create mode 100644 script.md create mode 100644 virsh_demo.xml diff --git a/README.md b/README.md index fa5b4ff..3949265 100644 --- a/README.md +++ b/README.md @@ -22,17 +22,25 @@ Computing Offload and Integration Subgroup (算力卸载与集成子工作组) 1. 无感卸载代码 +代码链接：[链接](./qtfs/) + 将libvirt从host侧卸载至DPU侧，host与DPU之间通过一个插件进行交互;在OIF开源libvirt实现，以及OpenEuler和DPU之间的插件 2. 无感卸载方案/代码说明文档 +文档链接：[链接](./qtfs/doc) + 对上述技术方案及代码实现的说明文档 3. openEuler/大禹/九州云demo联创演示及视频 +演示脚本: [链接](./script.md) + 基于无感卸载方案、使用大禹DPU卡、九州云OpenStack平台完成演示 4. DPU工作组背景介绍 +文档链接：[链接](./README.md) + 描述子项目背景、project facts、scope、与CFN关系、Release plan等 diff --git a/qtfs/CMakeLists.txt b/qtfs/CMakeLists.txt new file mode 100644 index 0000000..012c8db --- /dev/null +++ b/qtfs/CMakeLists.txt @@ -0,0 +1,40 @@ +cmake_minimum_required(VERSION 3.0.0) + +project(qtfs) + +set(CMAKE_C_FLAGS "-g -O2 -fstack-protector-strong -fPIE -pie -fPIC -D_FORTIFY_SOURCE=2 -s -Wl,-z,now -Wl,-z,noexecstack") + +# Build rexec and rexec_server +add_executable(rexec rexec/rexec.c rexec/rexec_sock.c) +add_executable(rexec_server rexec/rexec_server.c rexec/rexec_sock.c rexec/rexec_shim.c) +target_include_directories(rexec_server PRIVATE /usr/include/glib-2.0 /usr/lib64/glib-2.0/include) +target_link_libraries(rexec PRIVATE json-c pthread) +target_link_libraries(rexec_server PRIVATE json-c glib-2.0) + +# Build udsproxyd and libudsproxy.so +add_executable(udsproxyd ipc/uds_event.c ipc/uds_main.c) +add_library(udsproxy SHARED ipc/uds_connector.c) +target_include_directories(udsproxyd PRIVATE include/ /usr/include/glib-2.0 /usr/lib64/glib-2.0/include) +target_link_libraries(udsproxyd PRIVATE pthread glib-2.0) + +# Build engine +add_executable(engine ipc/uds_main.c ipc/uds_event.c qtfs_common/user_engine.c) +target_include_directories(engine PRIVATE include/ ./ ipc/ /usr/include/glib-2.0 /usr/lib64/glib-2.0/include) +target_link_libraries(engine PRIVATE glib-2.0 pthread) +target_compile_options(engine PRIVATE "-DQTFS_SERVER") + +if (DEFINED UDS_TEST_MODE OR DEFINED QTFS_TEST_MODE) + target_compile_options(engine PRIVATE "-DUDS_TEST_MODE") + target_compile_options(udsproxyd PRIVATE "-DUDS_TEST_MODE") + message(WARNING "Important risk warning: the test mode is turned on, and qtfs will expose the network port, \ + which will bring security risks and is only for testing! If you do not understand the risks,\ + please don't use or compile again without test mode macro!") +endif () + +# Build Kernel Module +set(QTFS_BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +add_subdirectory(qtfs_server qtfs_server-km) +add_subdirectory(qtfs qtfs-km) + + +set(ignoreMe "${QTFS_TEST_MODE}${UDS_TEST_MODE}") \ No newline at end of file diff --git a/qtfs/LICENSE b/qtfs/LICENSE new file mode 100644 index 0000000..9e32cde --- /dev/null +++ b/qtfs/LICENSE @@ -0,0 +1,127 @@ + 木兰宽松许可证, 第2版 + + 木兰宽松许可证， 第2版 + 2020年1月 http://license.coscl.org.cn/MulanPSL2 + + + 您对“软件”的复制、使用、修改及分发受木兰宽松许可证，第2版（“本许可证”）的如下条款的约束： + + 0. 定义 + + “软件”是指由“贡献”构成的许可在“本许可证”下的程序和相关文档的集合。 + + “贡献”是指由任一“贡献者”许可在“本许可证”下的受版权法保护的作品。 + + “贡献者”是指将受版权法保护的作品许可在“本许可证”下的自然人或“法人实体”。 + + “法人实体”是指提交贡献的机构及其“关联实体”。 + + “关联实体”是指，对“本许可证”下的行为方而言，控制、受控制或与其共同受控制的机构，此处的控制是指有受控方或共同受控方至少50%直接或间接的投票权、资金或其他有价证券。 + + 1. 授予版权许可 + + 每个“贡献者”根据“本许可证”授予您永久性的、全球性的、免费的、非独占的、不可撤销的版权许可，您可以复制、使用、修改、分发其“贡献”，不论修改与否。 + + 2. 授予专利许可 + + 每个“贡献者”根据“本许可证”授予您永久性的、全球性的、免费的、非独占的、不可撤销的（根据本条规定撤销除外）专利许可，供您制造、委托制造、使用、许诺销售、销售、进口其“贡献”或以其他方式转移其“贡献”。前述专利许可仅限于“贡献者”现在或将来拥有或控制的其“贡献”本身或其“贡献”与许可“贡献”时的“软件”结合而将必然会侵犯的专利权利要求，不包括对“贡献”的修改或包含“贡献”的其他结合。如果您或您的“关联实体”直接或间接地，就“软件”或其中的“贡献”对任何人发起专利侵权诉讼（包括反诉或交叉诉讼）或其他专利维权行动，指控其侵犯专利权，则“本许可证”授予您对“软件”的专利许可自您提起诉讼或发起维权行动之日终止。 + + 3. 无商标许可 + + “本许可证”不提供对“贡献者”的商品名称、商标、服务标志或产品名称的商标许可，但您为满足第4条规定的声明义务而必须使用除外。 + + 4. 分发限制 + + 您可以在任何媒介中将“软件”以源程序形式或可执行形式重新分发，不论修改与否，但您必须向接收者提供“本许可证”的副本，并保留“软件”中的版权、商标、专利及免责声明。 + + 5. 免责声明与责任限制 + + “软件”及其中的“贡献”在提供时不带任何明示或默示的担保。在任何情况下，“贡献者”或版权所有者不对任何人因使用“软件”或其中的“贡献”而引发的任何直接或间接损失承担责任，不论因何种原因导致或者基于何种法律理论，即使其曾被建议有此种损失的可能性。 + + 6. 语言 + “本许可证”以中英文双语表述，中英文版本具有同等法律效力。如果中英文版本存在任何冲突不一致，以中文版为准。 + + 条款结束 + + 如何将木兰宽松许可证，第2版，应用到您的软件 + + 如果您希望将木兰宽松许可证，第2版，应用到您的新软件，为了方便接收者查阅，建议您完成如下三步： + + 1， 请您补充如下声明中的空白，包括软件名、软件的首次发表年份以及您作为版权人的名字； + + 2， 请您在软件包的一级目录下创建以“LICENSE”为名的文件，将整个许可证文本放入该文件中； + + 3， 请将如下声明文本放入每个源文件的头部注释中。 + + Copyright (c) [Year] [name of copyright holder] + [Software Name] is licensed under Mulan PSL v2. + You can use this software according to the terms and conditions of the Mulan PSL v2. + You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 + THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + See the Mulan PSL v2 for more details. + + + Mulan Permissive Software License，Version 2 + + Mulan Permissive Software License，Version 2 (Mulan PSL v2) + January 2020 http://license.coscl.org.cn/MulanPSL2 + + Your reproduction, use, modification and distribution of the Software shall be subject to Mulan PSL v2 (this License) with the following terms and conditions: + + 0. Definition + + Software means the program and related documents which are licensed under this License and comprise all Contribution(s). + + Contribution means the copyrightable work licensed by a particular Contributor under this License. + + Contributor means the Individual or Legal Entity who licenses its copyrightable work under this License. + + Legal Entity means the entity making a Contribution and all its Affiliates. + + Affiliates means entities that control, are controlled by, or are under common control with the acting entity under this License, ‘control’ means direct or indirect ownership of at least fifty percent (50%) of the voting power, capital or other securities of controlled or commonly controlled entity. + + 1. Grant of Copyright License + + Subject to the terms and conditions of this License, each Contributor hereby grants to you a perpetual, worldwide, royalty-free, non-exclusive, irrevocable copyright license to reproduce, use, modify, or distribute its Contribution, with modification or not. + + 2. Grant of Patent License + + Subject to the terms and conditions of this License, each Contributor hereby grants to you a perpetual, worldwide, royalty-free, non-exclusive, irrevocable (except for revocation under this Section) patent license to make, have made, use, offer for sale, sell, import or otherwise transfer its Contribution, where such patent license is only limited to the patent claims owned or controlled by such Contributor now or in future which will be necessarily infringed by its Contribution alone, or by combination of the Contribution with the Software to which the Contribution was contributed. The patent license shall not apply to any modification of the Contribution, and any other combination which includes the Contribution. If you or your Affiliates directly or indirectly institute patent litigation (including a cross claim or counterclaim in a litigation) or other patent enforcement activities against any individual or entity by alleging that the Software or any Contribution in it infringes patents, then any patent license granted to you under this License for the Software shall terminate as of the date such litigation or activity is filed or taken. + + 3. No Trademark License + + No trademark license is granted to use the trade names, trademarks, service marks, or product names of Contributor, except as required to fulfill notice requirements in Section 4. + + 4. Distribution Restriction + + You may distribute the Software in any medium with or without modification, whether in source or executable forms, provided that you provide recipients with a copy of this License and retain copyright, patent, trademark and disclaimer statements in the Software. + + 5. Disclaimer of Warranty and Limitation of Liability + + THE SOFTWARE AND CONTRIBUTION IN IT ARE PROVIDED WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED. IN NO EVENT SHALL ANY CONTRIBUTOR OR COPYRIGHT HOLDER BE LIABLE TO YOU FOR ANY DAMAGES, INCLUDING, BUT NOT LIMITED TO ANY DIRECT, OR INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES ARISING FROM YOUR USE OR INABILITY TO USE THE SOFTWARE OR THE CONTRIBUTION IN IT, NO MATTER HOW IT’S CAUSED OR BASED ON WHICH LEGAL THEORY, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + 6. Language + + THIS LICENSE IS WRITTEN IN BOTH CHINESE AND ENGLISH, AND THE CHINESE VERSION AND ENGLISH VERSION SHALL HAVE THE SAME LEGAL EFFECT. IN THE CASE OF DIVERGENCE BETWEEN THE CHINESE AND ENGLISH VERSIONS, THE CHINESE VERSION SHALL PREVAIL. + + END OF THE TERMS AND CONDITIONS + + How to Apply the Mulan Permissive Software License，Version 2 (Mulan PSL v2) to Your Software + + To apply the Mulan PSL v2 to your work, for easy identification by recipients, you are suggested to complete following three steps: + + i Fill in the blanks in following statement, including insert your software name, the year of the first publication of your software, and your name identified as the copyright owner; + + ii Create a file named “LICENSE” which contains the whole context of this License in the first directory of your software package; + + iii Attach the statement to the appropriate annotated syntax at the beginning of each source file. + + + Copyright (c) [Year] [name of copyright holder] + [Software Name] is licensed under Mulan PSL v2. + You can use this software according to the terms and conditions of the Mulan PSL v2. + You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 + THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + See the Mulan PSL v2 for more details. diff --git a/qtfs/README.md b/qtfs/README.md new file mode 100644 index 0000000..a702f75 --- /dev/null +++ b/qtfs/README.md @@ -0,0 +1,116 @@ +# qtfs + +## 介绍 + +qtfs是一个共享文件系统项目，可部署在host-dpu的硬件架构上，也可以部署在host-vm或同一台host的vm-vm之间，通过vsock建立安全通信通道。以客户端服务器的模式工作，使客户端能通过qtfs访问服务端的指定文件系统，就像访问本地文件系统一样。 + +qtfs的特性： ++ 支持挂载点传播； ++ 支持proc、sys、cgroup等特殊文件系统的共享； ++ 客户端对qtfs目录下文件的操作都被转移到服务端，文件读写可共享； ++ 支持在客户端对服务端的文件系统进行远程挂载； ++ 可以定制化处理特殊文件； ++ 支持远端fifo、unix-socket等，并且支持epoll，使客户端和服务端像本地通信一样使用这些文件； ++ 基于host-dpu架构时，底层通信方式可以支持PCIe，性能大大优于网络； ++ 内核模块形式开发，无需对内核进行侵入式修改。 + +## 软件架构 + +软件大体框架图： + +![输入图片说明](doc/%20Overall_architecture_diagram.png) + + +## 安装教程 + +目录说明： ++ **rexec**：跨主机二进制生命周期管理组件，在该目录下编译rexec和rexec_server。 ++ **ipc**: 跨主机unix domain socket协同组件，在该目录下编译udsproxyd二进制和libudsproxy.so库。 ++ **qtfs**: 客户端内核模块相关代码，直接在该目录下编译客户端ko。 ++ **qtfs_server**: 服务端内核模块相关代码，直接在该目录下编译服务端ko和相关程序。 ++ **qtinfo**: 诊断工具，支持查询文件系统的工作状态以及修改log级别等。 ++ **demo**、**test**、**doc**: 测试程序、演示程序以及项目资料等。 ++ 根目录: 是客户端与服务端都能用到的公共模块代码。 + +### VSOCK通信模式 + +如有DPU硬件支持通过vsock与host通信，可选择此方法。 +如果没有硬件，也可以选择host-vm作为qtfs的client与server进行模拟测试，通信通道为vsock： + + 1. 启动vm时为vm配置vsock通道，vm可参考如下配置，将vsock段加在devices配置内： +``` + + ... + + + +
+ + ... + +``` + 2. 要求内核版本在5.10或更高版本。 + 3. 安装内核开发包：yum install kernel-devel json-c-devel。 + +服务端安装： + + 1. cd qtfs_server + 2. make clean && make -j + 3. insmod qtfs_server.ko qtfs_server_vsock_cid=2 qtfs_server_vsock_port=12345 qtfs_log_level=WARN + 4. 配置白名单，将qtfs/config/qtfs/whitelist文件拷贝至/etc/qtfs/下，请手动配置需要的白名单选项，至少需要配置一个Mount白名单才能启动后续服务。 + 5. nohup ./engine 16 1 2 12121 10 12121 2>&1 & + Tips: 这里的cid需要根据配置决定，如果host作为server端，则cid固定配置为2，如果vm作为server端，则需要配置为前面xml中的cid字段，本例中为10。 + +客户端安装： + + 1. cd qtfs + 2. make clean && make -j + 3. insmod qtfs.ko qtfs_server_vsock_cid=2 qtfs_server_vsock_port=12345 qtfs_log_level=WARN + 4. cd ../ipc/ + 5. make clean && make && make install + 6. nohup udsproxyd 1 10 12121 2 12121 2>&1 & + Tips：这里插入ko的cid和port配置为与server端一致即可，udsproxyd的cid+port与server端交换位置。 + +其他注意事项： + + 1. udsproxyd目前也支持vsock和测试模式两种，使用vsock模式时，不能带UDS_TEST_MODE=1进行编译。 + 2. 如果vsock不通，需要检查host是否插入了vhost_vsock内核模块：modprobe vhost_vsock。 + +### 测试模式，仅用于测试环境： + +找两台服务器（或虚拟机）配置内核编译环境： + + 1. 要求内核版本在5.10或更高版本。 + 2. 安装内核开发包：yum install kernel-devel。 + 3. 假设host服务器ip为192.168.10.10，dpu为192.168.10.11 + +服务端安装： + + 1. cd qtfs_server + 2. make clean && make -j QTFS_TEST_MODE=1 + 3.指定测试服务端的ip，ip a a ip_server(例：192.168.10.10)/port dev network（例:ens32），防止机器重启造成的ip变更问题，方便测试 + 4. insmod qtfs_server.ko qtfs_server_ip=x.x.x.x qtfs_server_port=12345 qtfs_log_level=WARN + 5. 配置白名单，将qtfs/config/qtfs/whitelist文件拷贝至/etc/qtfs/下，请手动配置需要的白名单选项，至少需要配置一个Mount白名单才能启动后续服务。 + 6. nohup ./engine 16 1 192.168.10.10 12121 192.168.10.11 12121 2>&1 & + Tips: 该模式暴露网络端口，有可能造成安全隐患，仅能用于功能验证测试，勿用于实际生产环境。 + +客户端安装： + + 1. cd qtfs + 2. make clean && make -j QTFS_TEST_MODE=1 + 3.指定测试用户端的ip，ip a a ip_client(例：192.168.10.11)/port dev network（例:ens32），防止机器重启造成的ip变更问题，方便测试 + 3. insmod qtfs.ko qtfs_server_ip=x.x.x.x qtfs_server_port=12345 qtfs_log_level=WARN + 4. cd ../ipc/ + 5. make clean && make && make install + 6. nohup udsproxyd 1 192.168.10.11 12121 192.168.10.10 12121 2>&1 & + Tips: 该模式暴露网络端口，有可能造成安全隐患，仅能用于功能验证测试，勿用于实际生产环境。 + +## 使用说明 + +安装完成后，客户端通过挂载把服务端的文件系统让客户端可见，例如： + + mount -t qtfs /home /root/mnt/ + +客户端进入"/root/mnt"后便可查看到server端/home目录下的所有文件，以及对其进行相关操作。此操作受到白名单的控制，需要挂载路径在server端白名单的Mount列表，或者在其子目录下，且后续的查看或读写操作都需要开放对应的白名单项才能进行。 +Tips：若完成测试环境的配置后，无法通过服务端访问所挂载的客户端文件，可检查是否由防火墙的阻断导致。 + diff --git a/qtfs/config/qtfs/whitelist b/qtfs/config/qtfs/whitelist new file mode 100644 index 0000000..ed537a3 --- /dev/null +++ b/qtfs/config/qtfs/whitelist @@ -0,0 +1,35 @@ +[Open] +Path=/var/lib/libvirt/qemu + +[Write] +Path=/var/lib/libvirt/qemu + +[Read] +Path=/var/lib/libvirt/qemu + +[Readdir] +Path=/var/lib/libvirt/qemu + +[Mkdir] +Path=/var/lib/libvirt/qemu + +[Rmdir] +Path=/var/lib/libvirt/qemu + +[Create] +Path=/var/lib/libvirt/qemu + +[Unlink] +Path=/var/lib/libvirt/qemu + +[Rename] +Path=/var/lib/libvirt/qemu + +[Setattr] +Path=/var/lib/libvirt/qemu + +[Setxattr] +Path=/var/lib/libvirt/qemu + +[Mount] +Path=/var/lib/libvirt diff --git a/qtfs/config/rexec/whitelist b/qtfs/config/rexec/whitelist new file mode 100644 index 0000000..b7f34b5 --- /dev/null +++ b/qtfs/config/rexec/whitelist @@ -0,0 +1 @@ +/usr/bin/qemu-kvm diff --git a/qtfs/demo/Makefile b/qtfs/demo/Makefile new file mode 100644 index 0000000..33a9e4d --- /dev/null +++ b/qtfs/demo/Makefile @@ -0,0 +1,12 @@ +CFLAGS=-g -O2 + +all: cfifo_r cfifo_w + +cfifo_r: cfifo_r.c + gcc $(CFLAGS) -o $@ $^ + +cfifo_w: cfifo_w.c + gcc $(CFLAGS) -o $@ $^ + +clean: + @rm -f *.o cfifo_r cfifo_w diff --git a/qtfs/demo/cfifo_r.c b/qtfs/demo/cfifo_r.c new file mode 100644 index 0000000..02e6258 --- /dev/null +++ b/qtfs/demo/cfifo_r.c @@ -0,0 +1,98 @@ +#include +#include +#include +#include +#include +#define BUF_MAX 256 +int single_read(int argc, char *argv[]) +{ + char buf[BUF_MAX]; + char *fifo = argv[1]; + int rfd = open(fifo, O_RDONLY); + if (rfd < 0) { + printf("open file %s failed.\n", fifo); + return 0; + } + + do { + memset(buf, 0, BUF_MAX); + int ret = read(rfd, buf, BUF_MAX); + if (ret == -1) { + printf("read failed.\n"); + break; + } + printf("%s", buf); + } while (strcmp(buf, "exit") != 0); + close(rfd); + return 0; +} + +#define MAX_READ_LEN 65536 +int my_epoll_read(int argc, char *argv[]) +{ + int *fd = (int *)malloc((argc - 1) * sizeof(int)); + char *buf = (char *)malloc(MAX_READ_LEN); + + for (int i = 1; i < argc; i++) { + fd[i-1] = open(argv[i], O_RDONLY|O_NONBLOCK); + if (fd[i-1] < 0){ + printf("open file %s failed.\n", argv[i]); + return 0; + } + printf("my epoll read open file:%s success, fd:%d.\n", argv[i], fd[i-1]); + } + + struct epoll_event evt; + struct epoll_event *evts; + int epfd = epoll_create1(0); + if (epfd == -1) { + printf("epoll create failed.\n"); + abort(); + } + for (int i = 0; i < argc - 1; i++) { + evt.data.fd = fd[i]; + evt.events = EPOLLIN; + int s = epoll_ctl(epfd, EPOLL_CTL_ADD, fd[i], &evt); + if (s == -1) { + printf("epoll ctl failed, fd:%d\n", fd[i]); + abort(); + } + printf("my epoll read epoll ctl fd:%d events:%x success.\n", fd[i], evt.events); + } + evts = calloc(64, sizeof(evt)); + while (1) { + int n = epoll_wait(epfd, evts, 64, -1); + printf("epoll wait get new %d events.\n", n); + for (int i = 0; i < n; i++) { + int ret; + FILE *fp; + printf(" > epoll wait new events, cur:%d key:%x data:%lx n:%d.\n", i, evts[i].events, evts[i].data, n); + memset(buf, 0, MAX_READ_LEN); + if (evts[i].events & EPOLLHUP) { + epoll_ctl(epfd, EPOLL_CTL_DEL, evts[i].data.fd, NULL); + continue; + } + fp = fdopen(evts[i].data.fd, "r"); + fseek(fp, 0, SEEK_SET); + ret = read(evts[i].data.fd, buf, MAX_READ_LEN); + if (ret <= 0) { + printf(" >read fd:%d ret:%d data error.\n", evts[i].data.fd, ret); + goto end; + } else { + printf(" >read fd:%d ret:%d data:%s.\n", evts[i].data.fd, ret, buf); + } + } + } +end: + close(epfd); + for (int i = 0; i < argc-1; i++) { + close(fd[i]); + } + return 1; +} + +int main(int argc, char *argv[]) +{ + return my_epoll_read(argc, argv); +} + diff --git a/qtfs/demo/cfifo_w.c b/qtfs/demo/cfifo_w.c new file mode 100644 index 0000000..85aaa38 --- /dev/null +++ b/qtfs/demo/cfifo_w.c @@ -0,0 +1,30 @@ +#include +#include +#include + +#define BUF_MAX 256 +int main(int argc, char *argv[]) +{ + char buf[BUF_MAX]; + char *fifo = argv[1]; + int wfd = open(fifo, O_WRONLY); + if (wfd < 0) { + printf("open file %s failed.\n", fifo); + return 0; + } + + do { + int ret; + int len; + memset(buf, 0, BUF_MAX); + fgets(buf, BUF_MAX, stdin); + len = strlen(buf); + ret = write(wfd, buf, BUF_MAX); + if (ret == -1) { + break; + } + } while (strcmp(buf, "exit") != 0); + + close(wfd); + return 0; +} diff --git a/qtfs/doc/ Overall_architecture_diagram.png b/qtfs/doc/ Overall_architecture_diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..40fd7e28707642801ec0b984690a25c08e092ac4 GIT binary patch literal 49089 zcmeFZcT|&E-!>X-%&5qW4e5@6NSAJ?jwn??=^d2brI!%E$|y}jZ$ZSsP(ll#*FicY zLg+0hCDb4U2q7dncbsRQ_c`nR)>-HK=dAO6=j^pyxk>JOXaD-|y7rE59~o$|o#i?U z0)g0cwC@{%K&QJwppz3nodMoqDbHvIfv$se?*DEQnzK1)?`BJAQ9SgusoT%YY0SMJ z9Ow~uk;E1J^REXpKeJ+b|F|C}Epf*p!jJh1I{V=E1?KDB{GYm6m>0J$cidsVf?nVF|0nt1M<=sm-TfyyS2y|& zwp2Nfo}I$fT2=61CJ0!jC&#oTV)iZVLs61I+OtK zO`5Mwq1F;69~l+r1t^M=_m?IgIcJh@r#*+yft&Wnn*G0OPj{agK{css(3%27v`_)_ ziK7_z!PW6Af99rEPm)w}FSU3NNq*;(O?%$T;UBz9y#R{7QT>z$RF6ubj-aTI-c-w0 zz+2y+$Q1x)W8(abc^at2$^6+8^s84_<#GWbZTt$MOy|&KsXJ`?s!t_Q=3sg zhMb?-Adi;FBJ~Qfg2vxO60Z(`+pt#__;BbbD#l5Dht?BfnDN|oOKG? z>6|k9L$EUEYre}wrHTDg*a;A*P8SHfS#We@osesp1f522!f)nWIwj9Bcg@;`GFUJa!lc;Q=BRiP&{We3XwYJKr=IOhcS-$qrP zYHg8Pjzi4eX>(l(MUx7l{3>|U;eUm4d+pU zx5gf9tu=MsoEc~ z7IWW<#YR2-*3}n)&j_Zc7FXFtxUTPs`94%ZdCz>Vb)C-$EmHfQ8e&&wzuomVWM%aG zx6iS@&C-6(-}*muV@K5$!K1Qh?85|Oe#ngMC7cwvVBFQko*Mtw6`t9MSG?LbEFxTn?Z)~$Pg9IgXaR!OI z&rY*)!1J7E@yb#@*$U~{ht1L(b6I7KF2$SkVT)aFIf}-DP+)ja!MX9KQS!6m?LA#Q zc4TF@|Fg2M3aT9iKTvI4pw3=vqqmU0jpnf$Ps}I~rX)P)S8S-cZ#nY9{J!OiDbid{ zffgiPx&YtMT2t{?VhxcG{-ez4PsXEQMKe~Z5}01t4XtoxW6!bKGz{UwxY-9o&moy$ zqL-+Su5w<+>?{gNYTzl$&G_m+61H?tGN(LBA-#I}?R*VF;zz)|ows6;s-8E0$>ULh zI!)hc30rn3HX~Oxh#3z+V`NM+8>-x_U40EU?ehLmt?rp2D%F15(0PKs zXF$xEPQwUPj1ua05SYDIqda{7vQ4Hh5G|YGK_v-7@l7wt6Jbk#)ijhX?FzKW&LjJd zVlXS6!U;>gHI3mL*Z>g{Izy->#_~wH3;Q)$~T0FxZ_#f zUqKdT&C_$WF7Q3>$Wa^lvw_}XoR^ocVa%v@C?+XXR6XC@hh`8AcywH|3%5ZavJ5k( zV%+t%lP-a2>*Wt!;Hmk0{L|3nS^>mn#8b2PV+YzTTm2@h;|JPXV%GeY9;zhYcNUQ? z>Zv1`!3L)S-oedZDreI*G<4P$0s>w(V9Kr8QH0Pt)@hfn8lpBKR=r=UERi~ee~lGO z?H3*(@_DjxB_9FFZP&6?AM0_3{8|?oh42as&3{Y?Q|Ft9Sx20TwFnKsEVtq8k0a9* z({qcx*&g*xO*fDR!Je@ z;57C0T-zpCe!d9?K47t3v(TWDqlYjuDjrlZ*0BP3mF_-+F}S;4QzcqF7Fq!Dv867;oK z;u}aL!QO@RxK#<0JTH?te z!qlhRd;YV3*#rkWo4=7U0cQs|wyb&U!K%*l=pD3Gt9LfB58Y8X_=-pgR!grHTpmzt zczg8jz`AWr4-6@T=?aBCS$LbX+QNYc+BFxP?rBhcKpdNsL#xkE-nr-l(PTD+Cp6i2 z=LYBM6nrj5(s_f9Yk2leJ6zdU8fo`I{lI4b`i@WK62wXynNC_W4?&nj*`(&vd8X%* z!Mw&s_FLKRw9+2Tve!!Oss*wB+|ggas;xSh_K;5})fjUsr=zT!My3>~7F#g_C~E=R zthrYaz;ahoFQMrhsdSDsvKkBM*~CSl!wgA5#R|@Qw-{$j%zJ2?5odDgx<*Qe{WWZ7 z(o2VXAjCJ%WAp5NrTnz4h^i9hNRqaJ+>1>?9U+?3Ad1u3F}YVa%1&xR0X7)X@+n=J z$Rh@c3W24D6{RnZuT)nJyDV=u#0iPCMJg!gmC^>HyVo9T@r3G3giNtfKcm z)nrbiu62~AU`}^Lib5; z@)V*Xhn%h9lT(xT_bsZU!lQOED&~-%W#r^a<8>El8D!o+X<{#RgNLXmQ=Rb>NjS78 z8}Bwwjo2~J z)JKwA5k4;M%3+a0v?j>Ng!aXre$G{8vkF;Gr5aUL)syGS<9Nl`UH|kmG+z<%+MIP# zCxN$It||wMk!zxL8N|K4RAKXriK(pqUbjSeZB|L~?Q&D-UPF~M1>$C{JkWQPs&6dM zNRg|yN%jr@xGIR?o`mTDEP+K!!zO4O>M9gcrSxa2%yZ3O?C1Ou3c&g zu#%5UW!*ihqMz4KYY<~4tEbPO4p+shwE;=V4!o;vhmX9afv^SFqr?pNf-&AdODnvG z1Izbi$BuO1z{4{N+h9GVgVWpYjsEub{!0x`3;R49{3W&(CjI3!#kD9^;C>F6`AX)Z zbE5K=ohL~-U@#kxx1hz`M!mbm3oK&aRlccxq$4L1$WN`VH5+3#cVxs)x2}-}Z^#eA z3)c1}$+`5WWYlp8!fazIm(&7@>e{K(AhB(Zw!;VIHVq@2GN4=0G5nEt)Q^S} zvupv|w^#qv{gp3pxj!{x8-n)?<^)}CuZY|)bx8;8Ug^d3D`5NwDI2qp}W^zLJzoYtT78Z=f9lQ6V}sQb1a4>V^BX2UUtQM)Dzl7T(~ z*ReQC2RX0bF~E??Po32TG)m;-<{zl-@obyh+d2rlx5Fu1M4QR3UYpsv$7G`%&M;LA zjQ^sZ_O7vh$S7|~Rm9&(4ym{~wHFGf6EP~Os4!Wq%K=~R#C(*PYm}|J;y$U3;RVm~ z8upi%nCBmwvS+Am+tFMizfWV9h7@-PLMkLj&hY|syIlpoq$^+xdAzxdC(NiHy4yj# z9ATbe4B)ooCUepuJ3KdN9lN`m>AS+J4wbGqYX+DlQ-eFbmLnR7rw^95f zDL#vr@PiQFcozZ+kHtlWf?Z(|IX^g~g$o~3&w^f)=u@%{M~5M#(WUYnZqg+4wrrQ& z&@E)M-_5Y2BCLVWLc>l~(Jrk>QK%s+B#LHME)=#oHL*Kwm}juH7}cweR!@ze$|#E5 zwX&o38E2wFKZIanlde5t1!)|;W!GoDw6+%#8&DsQ3EAiiFVV2M+0N7OjG z0df4GMp_KO`?d4x16DXo$d4*P9;%UcvAfisw(MfD<|F%s&2@jFXuFb%bVkA&UTTCQ+tOv3K3Iu)4EWQpO9$o zNP5V;pVb|4qFr@kQQgjVK^w^+8%BK(RT!hbvxkx&MpAaME@*Qd|NTVDVenOE>Mc_O zd_*M5NNOPASJqtB_xZ@<_HA#H$FjrjnlO(lB&b!h278ouZ zEM;YcyeKt1b)F1lzFC)vUAna^X#?MuU5Y!HoUL&;Zl%m^6Q&-(*2wq>*|1>O-SMg8 zqroQhT?npFrL~<<-oh}$(2pL@d)v6;7Q>F3lu4tG+{rcr*6I+jtIIa2VU4+V!v|`Z z7NU9Ek?GOV5lql?ysYmeyULfPcYqdr3~pZ_Uk7Yj`h#AcjR)eGsqz0W6Vheg!7T0l zki)Mrq*8lY$&@qb5hz>H*C&`600Y7J&!0cvN5z7C^0*a!mZYL6dZ+==2hez#Wil&Y z=m+3-pu0?Ab}+#BuR=4ddz$FSpyV+!8Hb4sw;zQKN}fwcdwM?sB7TW=?uVFtwr3no2=Ns>1V_qsE+XCWO}^b#9?pWP z?2;~_a%)PJQoItSw#>2_asya$<-3yNxEw>uH*{QLq9L>aEx(l*?4nhik{0LmX*wxE zv)z`rZ>ONsJ*UO5*qjRtPwrgY$D+$e*?<|IvHt=zf2AL-z^TbE3pfour^&=VxRKsX zj%Vhj#X0Jk1)VQ-VB_URN?cmjO4ZxX^bA_$GKRynd~|wRJJ&L3V+qiyiN}^p0l;LJ zQ<<#wR*67QKwzkf`&R#RC~qci_!B}1L4<;v3u=`j%G12u^kvkrKo1VGxS8#H0#s+d zY$9>l5ORRwcxN~Os(ihjNhI%Yv8SaBnj_slS+qpJ%AMh1I`))acy2;RTn_?{4LI z!B8LtE)YKeeZjMAlK@(LV zi!TdwCY~1uSnGnN+jMV1L}=#%Db`%x|?h&>woO_hH!4!+7-D1H`i0BxoH{v z6CRd;IXKUr;1Y2dA<8?~NBJ68ByWpKtE&qPETnyN&{Xt502+HH#k9BnI4l7ggOe7x z?yEoX9J*FN2M%1bcE`GD21tx~$Ty@3a^qwOjflW)<(%ajHC7^T8)e9MHyeLk$V^Id zb0Mvy+z5g)-aVL8>>hJl<0j6`KhfS$zG1C@d!zu#$;Xd9{<^nSdlqRQ-89m&$8i?~ z@?cg3pX9Wfhr~DFEOya>L${4L*{@dXaA*_xi-4x;i?m*C9m6WQ~y z5ePH2dU)E1$JL*Ja=CS#`ABJ6g6h{oi4|vEBt(MvG0)kG{QX1aI<@TkyAH|&iL2V~)k&f9W4b~F zDalHzGg_uWdIh~+=B>h^oCf&1ufArFY{Xt@b%>#ou7plWagI{f*BaOD=fvX5CWHW_ zQF3MVd#H?BPDeo}uG<5!22o1x5+L9#=f7EnXLS%dg+Tg|?Bbg4%BLogC=1`vHr)K! zy=%>Av^=+>T9)t7PlL%OTK>nY-vu=xG2bf>!qJt*mwzLBJCpmc&l%C><( zrC>r@aZJghAt??#S?%@*%*ORY7&Cn`FaGs!6&QtzJ^6#~1n$%6 zx32xv++J0spN(rHY6eFh>rBIzP^I9V;UW>_Qz)p62WbM_2ikRbGa+nC;%ibTwAM=Y z#b?^nm;4PP{HbCm9hNPBkTsX>!Uk5(ON0j6N>=E>+LnLa`piQ7`N!#o4_~?!QvKba zs8W!FJOD_531>Tr*vXgd%dgosj4jQnx@)+Nu;oSWFepib5;bw`8c9w)dR~Uj_nz@+ zqwN*g9ERG4*ZOEX`yd-F?ZKr{NZ1&3)U{&+gpHe~#mrA$VQc`_p!GD_pT!T;m54wM-!0nkeO_xSRgfZA(Q2-~y(^4nQ@}%f>5Hg)J8-1^K-xz7 zdXE}Lz#gBomF@e4^GK$fYsRFXIApJHPMbG~0_+vEfonsjw6Ydq#C)f+=CT!?+-xN} z_~!TGRU#+_Jk3dhd4x7I$>lLaZdZJ7hu-$kPYcRYZRxlP-@)@;>Cd*hDza@X>6@Uk zYs?l3Xevrxf1_=v4V9)2ta+6^rEezhMFQV*rP+v*&sH1}uG*6K&`vNHcVb~iW}KG2 z``4SL1w=8#le*+#*AOYt*LdE0GNtOMI5!;+NE{EL>((A8mbwqPt|3QH&@o5y!4gIj zsTJh|Zp@BWQ1%v$K|eqZ%T3xnja9(W}LeikpmKxOyW38RJHn$Z*vP8~Uf(>Z9ZVu)lS8 zqdI@2P+ue4REu}#2?6B5a*$4%A2m1XNY1iNEwutF;7IYRHXWQSWNQed6&&sRV$m#S z*-U~7RNph&IWyavX6@!$ZR2gO-rrH-t}=}0_^pIBS-5Gy&DzUMgPuQKfc1$ zcq1EnRA6Xa!n%{X&+YH#>29W%a)r-x&fGlo3h546umvgD&!Z-u0f zOF;S#s#EK{bi$^i>Y)B$;d$zv7#ttRG~xMTPN&Dix_nq$_>*}v%~}y$*uUfkyiE?TZU^n&|?KVZU^)~ zekYtxS}#(B0jY%Z>lTd@@_nO0ZgHvmW^Vla64}*ZVANx^inR@%FPDJL%Q08*(#NK; z7C~K%6Oe|w6YB6w0A~a1KH3CAz|Qwxs}J;o_gvc6%z9g`4+f!*Z6;1Cwp!s2NiJWb zJhGpS$IR~+g<>fRJ(9A4`^Kpff-#_~)z%g@w#>tW+e=kiv@$db+s#&NjapRAvo z6eA?8)f-bTnWxXZf~+~BZIMfRS`(;^?i#82LpfI097xm|-I;2L#v&hJ^Nm_BDTyy< z5NiB}#}rNf#c;T_!!miO0xAZq@$Zvsa!o)p77ghmVb${{===8Bwr46ou;S{x1K#)x zuu7SZc%Ibx-~eZQWyeiSqzhW2T=e-u7oOiVuQ~d=;{3O2bt{cJF{X&O0V1|iK=AIb z$RDVi`U5{o&t!t0QPF7ha*sOQ52&}%LO_m7(=dj(7gj&H>JO;)PJ0rW3qk#Io!sewh8MH zo3>kzJJpXh{*yS?7Pw}Qa*1cgb-kF|kD~N{>+oIwkJDEArbwry2r!3M56W-*YhZd> zsMpT#KQKU#HLf)TFQXPBTwEyQueSVkI8Io{3OA1yp5v9!FK3<%Y`xQQr`@B(%zxbp zT(G#fFI7{BrVTUuRuKWPsmRZK=EgySJ&6~JcH|T!_a{{=u5Q}T)+<6TgcKpvI0`zo z0wh?gvC4pi?Y02HQ)wky5s{uP? z+)_k?n!&o`)*5D0l54DATa@2s@VlFb{3lovY5Dww`sDO{s_GoCd#0?5=XBG-?6hV9 zO7XV0x4t<69ypl!Wx4^kQJgyu$52+HgvKf$et|07WIjwyvF*2?LsChh zRq;{;<*8K4SZPOqjxK2+1|s=DN0LA4{zRxPTfByVF`W0d0xSs zgGS)}k(#kKr2F>zU_ye?%Fr>B_uv8CVdV|X_~Z(r!)g0@_%%x=Wx+vOo!4Bo&`0n^ z)?$Bh729VMnn97j&$97)=2N@-{7O@ka(H+!p;pW=Sh|yHEQL_X3V%PHRbH+YpenHs zVQO~Rine7t)$}!7&21`)7;YQyxP`j6DD?1Y0sxj5=TD-m3WsnxRb0ROdYT^)*Z#y0 z6!Y-%=)25PpYG4zL!wTstmT#FD8YAE50`S#A*xw6zI_SA=^T0ZgvnvcA*#fdxPtXu zX_|l^l(S6@Kh|A796n!EI!=!CfN`{~zz9KMKXeuX2Usd)C={v#llL57dd|w{=KtWC z>k%%Gp3=skZ3Em^^i57aK1>^JrfnI>j@W6mCke|q#;&cedjwm)|5VT5uSh-HE(+jK zZF7jDaL-QM1@@KI9b?X769RnJNVf-GUUReWaDOmBQTzD;a75)J!)&P% zgyVGcpvn5Yq9bO+NH>H$nv)u=VwHz_ZA~1k{7f)iSIg;SUB?PcROgQ-rPX|L#DC?1 z|N2hqD@-qUVp*laxf!7ApD7;AZAYeK)fp-}yA(nmA*79Jn=Ez$KrI&X5^JmT=_7Rp z%S#Jx-ecbn-QKWj$<9uz)wSsAX(`Rb$lg)P+U-q9aoJ1$bLr*qK}0EJTu5_(bLw%{ zc=*Sw_wuZszD^~fQ`;df5{7wRr)49hQ$2cAkP=p?Cmq2YB(HCi)xV*81J*wv zb)(6XUkiEys#;_rTw+OjbD6n$*ov%mwBTYV_-N$&2|^s<=;Dc)RPze&`A2mrL}(0Q z-mzFYhTRZ~wn8o)_xWq4CwJqAv-X_T=Std;)s9ezd?K(2lfBw+LMNNA=t zfGaGymE3aEzFX z4qlGI1g37Z%xs72N7yw?HA@o0>Yx#qmUmXt7o}I})Dr%ulEy8Aqy4V|f$) z%%r%H5V;Mi=Qz*Cw!A)8%3KOckLB#sr%!Kmc6OfgX+36oFS2glYPlr(41n#-`nuRf z*KQ594SgN;*j}36Tp2;|u6ghRs(FSnSE?=umktZWz7X@Y)}5O&UoS_l^70IYcFRWi z3nH@1yS?|i?5v_Tu8c9sMCGf>`F1m*l4d1q85L83&VmAK+FHXF=o{C#xcULDah&f?p60u37OxQioQtTm z(p2w_i0$^Y8mX>2Xo?RADS4>n>Ns{-oWnn#feCxj4U~J??uvaGeD;ALXIn&Jsk1?ZMA?B{Ns`BRq1C=UJXr zxV}1+NLRwoinA~`C-h;>>~HF=&CN6EBi_xUU&Q;a-vEmmeu`T1nWjwAqy5FP(ZVqO+!p}82?6jSp(EJ%MmZTLJNf97Lr zw7=-GxSSOE1;rXRJ5NC8)XK+>*1jSH2?KFD%NUTeE^35T}xi1d7YeWc>Ya15Evz(wCOI^)s-)2~b07b6?od;rC&l zwpK+*qn^8t5i;(RN>)wqR)zx6Bs+FvOi*GhE~n(v=v-17lgVt%DCk`L5$Oq-Yi;O_ zyNyLr`9?$N>C}K0tD!xB!)~#|KO|YH{b^#HRr^=HRfC1H0yAxCt!>hny{(Q24Q2zWg<*-&k2QzoNa36|ucrKCG% zB9@vXWk0wYs;cwQ(`traAIJb<3ZpsT?w#a5`04Q1<_5XCjUj>p1YZ{^7O&^+zx+=|VrR*#ZdUxG6 zAF52=aQh3fa zvWCyU4tRzMfWJJcLWpI>Zx+J5Z4YENH!boivX?KmL?XV)dBp< zWH7!Rpk%|yfz%%S#F#LuCaDgXQ9=S&U))c$5?!=7O!)%^X%|3DQ-3oqGg!bmGm<$eVYG_aid z`{z{L-77=c`?CR9gw2Y7!yXkth7Wvz8!Tdb|2}I5%zOI^prn5d~QkvXrit4klY!=z7L?#v`YEEr*EIW zCkh;F`Z1!5fB8Y$GXKapTWk`|My%@LGY$i%D}g?I@9*#bXTr0?G7c}r#KZ*tS=r~` zm?-zhXgA5w{MYqTUx8L);QqR43kjpFmzJ8Iz5xMFA;5YvF-u_y}wA-w1|y7?BrjMQ9phH+>q?ZW9Yy$`F=iJ_D?)**Eg+^69l?+*by>+ zD+t(7(vF1)fVAWNREmGF>`Z0e)c8kfAP3P;%$-k8$oDC4B)T9ZuDnK-Ev&GIop z5nn_+ogna}$3q+7sdLqaAj{tNGhWsGSO#*|*)o_0)i(-iGRAZ}kK3Bvtuw+{NTqC*)F79_ zo>0n#u_@`Jt0q{7^l&Sg{Z3tSgV_~o4h!Yq29=Hvme+bw2j_n7Y<6Kat;tSH=FCgf zB6n7b>zK938`dm#UD$*15~Dgq%{Uf(GVR8{Tr>L4Re2TTC&z2>zMPwm;hLnagOp-3 z|6jS#LaMMxHdAopdkdoSV%`Gu462V4t!H#flj#x;HYZxi2tZrgFi zJqhxVkvz1@Um+p=%^n;;nf5mssF{TS93sA~gGSnE87Pfaj#$p?bjeH2;U4#5Zu>PBlYsT zGQ|ipof+)!skW*wgGIcp(So6N4S`~uoX6L>9b4${~3y zY{qS>darxdXrCQWTpHmdzu-|dC}<%__}ga4wch=MeO}AL+G?zB03mzQh(PI0KQrJR zG8K?EQ+4=8a=pYlNv-s~Pk4~dRo-=mbi`Ti%x7p!;n+oSdn_ z%kw%gX+8)=)0fdQH*SV>;7Z4R%G?O-->0(rRyrpS5p)H1-XMdgNkUdYWF zYAEFOr_*7QkSp)3=M)tXQ#ZA2hK*yn8gpYQU*iu(`@>|y`8pfZpVqZRew~mcNEbZv^2dXw{>jPFh>}gE5vSvZ znx9-PS~t?_KW5LEHh3-6-02cUHpMBJ5=D<&jRp#((stwZ-N()91jk~7qHF~#RR!=i zBgwOT4Hx-hPsegdz_r@S6JnAT94;HqrK`*+WpAxeEd+J0lsr@(QtR3%8o5XrIo-?q zPZoY})QQ`N&G??uO&ss`*D!?0KQ=CEh1E*NYZe`y$&F>O4V@0$dLf)f2-iO=7N}+P zOqV8HI8uiCKnc@H3i$q6((z#b^OV@G0ywZ)S!SSdxH#dHs{)Ve2M=aV_{=U+(*R0*Trn9g<-}lito2??qx+$;DoXiG%0`maPMPYF0xyzYSUkthy`x0vHOood=jNb>&G9EU@=$vg$~90 zrdLf2<=#u^Sbh2Y`f}H;g45apT_zl{MdiVO?_B+m0Ze)OXUFdL%uD8h#qY25lH7sn z^W_R#PJR7f3Ng#vupR0#pur#iFF=UZ=0;xu?yxyg_xzs?gVnOXb`SKCK=((6|}YOMy5U}2psW}--O5n$+Brzj&U zZ3vpnNO0NWO9M-R8B*NwZK!%#cwV$0(k@7BSE8*Geb=}6 z-`iAbPb%3~etg?4zW8~MFNw3aYWv(i^1}Or|RORq*Z)k%ZjJb8M5je*J)3O0plKV@KhN{=FlM6h$ww8b44dl_sfs&F7&> zewL}WJu|LxZ~1<2dxJf?q+mTaUn}@tRavIwc~ey%C*aNHjfnM{VxOnbwNNEmhJ4eO z%N^GSzJb8Ka|sA2S5{(6q>p~g*!%C(S<-2zU6-;^MRXaEwj(KzHkC)SEaqmp-qzjI zv)|X9Iyh>xsCH0KA0o159aCmtpA7ljN80-K^2t>_XpyDItFlAuhy0|SOO4Uxw4zG5 z*PQLAsSo_?t%Tb7(Cenq?j(xErC0lg_?BWtzT-D)VTZ=kw5r}t)297Qvlum$t0&7r z!__O)2=7yrbyUdSw3x4lflft-^Q&$~DV^@~(jaQ9u0|rVqojg{toDH`!Wp6HtP5s0 zebgJ-2TtnGqgQ*M_Xq(lzL^r4A09*>a~-V=J#XziM-t(ETSHi6gDjR8Bd4L2_s%ig zjnt=&>ITuZe#cLbym?hZAHht_%nI>`@%USQv7qZMJ3D^pnwpeG&RZYZAigFSti*c?8Dk&SYQNwt~=2kDvem$N)Om1yTkP4HY zGTWk$-lRmzmD(TA3fQj}D2?)Ff2s7w7S)JdeovNfo2>xOhf-42D9y&KWJakSWLZCT zYJGn*01~vy9;w$B^ufSdrEb656XoA_@ylNcqN=2A^#Z8%34dy2aR{E4TI-eE9JQ@K zymH>L9$ayzRiT`pe!MlQq27kjr*-ibLCH5&b8^RKU-rJN4VRCiEWoJK z5Jmo8di=|hJVw-BKZ7n{8`&j5-fUY>w)YulqhHyTINdlz`gf zs#nWOWc$FcF{p6GRzuZnC-uXMqc(%%JT6+;=02)^)0lLGwdbB-hql?+9CRnDWSuv? z-%YZ(uo&tRchbW5YSe*Hn9r%v;S6`P!p9=CUWdrte#)Dh%a2suH1Dmpq+)CMk|fl z<8+d~lH5H0~4lt@~;M&y!eVHVI3z|EJYL*)6Ax zoZ6;AX^AND%&4(x>B-cChN(TF_dj1!Z@h#o(7PH@Aka{ zi1{w#=ii=m--!Jaq7SA%qFuQiiAi3`GbNZ}-Tf1>8 zzsD-EZW-N^{ANYLG{{w1gkmecUP_vE146yHuioM5>nyC6>mQo_LafH(u{F+r@rwOo z;axhBbof+IEXhPgHXk|M-0x2P4Z_SIws4qmxEJ6vd4ZC-oXUhTphw^L`gf0xo7{#S z=)Q$!*l0^vKg_JA;z)D-+Ue$$85ab9AlX1b}VJ zw1zzhAP?@6SkytZ#H!R?4j{M>HVja=d8ia+ON9q6g@L1WH99?v=U>Jy~8cgpjW zM*PzkKt+y|dN_%|88+?hMyrdodmi(YJ()((pV!|I7_R+%g>>%0IHy|aCvSq=je zbXGopweu-UiFFk`B0*H(m*d^`l$)40@^~EAxf*Pk{Ig)q1GvJzyp`bVWp4kqILPk_ ze6D5x>iAEzioCmF^L^E)cKnKt= z9ti~z&u)wZJ#+Zh`X7ph|Yxq9j|Q;Fb4f)upbL3PiL`1*YDi z_Ho?EBwa&db>Dq)XOe=g#@={+ORT^<=iD`dkAn$`v(+ina9y?_*LXc$=xTG~AErhj~~pub!|-&iy642?^TJFQ?YUqiyLgDTpjWK>v;?zIvSslUEG zRGBjs31))NCChL3Idw|R3D;kLV|i;L3K}ndQ^0B>q0=E^n{C%P+nZ^|&2dLXT&rX& zFr;`p!pZtkmWWPH%i)c7-?QYQX@dU)5APTkaroYVCa z4@(9#FT+d~Mx-#;dozi3=GrnPomx1}!|p=4nWaCnb-W&H)XDni?0A=?lAQKz-ULZ? z`9Kwq{r9d>2g94fl>QTcTs>KkW-&7-dv*&zljXJ;N6oDG!$=+Brp4+%^ShqZ2;E_# zi0Xo>3BOxUGOM6y<^3KpRYrgv+xbJpLG7fp*_dRZf=*Q2xh!+q2;QtLQeURfJ^*nL9b8k4Xs5^YM5>u^Jb$KV=2VeoR*r9>=(W)O(&*f$ga}FO1jK{T{nsWyHJmB7KrjO{M@oa3sjA zPJC|v6O$8w2#x=IzhXbo{t%E8^kDc0M{;zgX%y#;eRAUG%AF+L)9sC8-ve8g4!Bvm z3eZ)}UvPH%qoc(YpOAV5!^N!~7A;#}Z0=4-hCT(CPn1>O3%UHWr~$&#^>-2pR`L<$ z%pvD@@UB_^0`GRh;DUExVcD&DCIX%_&f?wapSYOa@e3EZ7q?^)VNgCZ)Khg?L$c=z zbWTb~NJh0F%=-0Z8*Li}Re?mLL6LcccoJQgv%cI^%lAlh$1?L=Q4TVlL1_JeJ!8;wOI%}z+|^&{PG4p31J-zh;%8cd}+8mf54xVU8ko{y5xmrN);dQeK#qSFp#(BAG`nTcN9@)zbQU%ix=!+ z$a~#pkgz#{bT8t%@!&V;1Y#u_Er$2uOF&%nS1RvT0xzWt=T|yfln;y~C103RQqHUt z4^%8$cQp&h8Rnex)V8aD#&R7@tc?$ek8k*x;)f6KeLS~&zP=_guYeh}?Ts1$nYNMP zEZwuS`G+E`;?9HhczGf5iH6v9{+~*BEwYcobNv;3q=E{n(}S#j5-_B$!NX0pj6wz* zi{v5>bd4I{XS^v&_V?F%ckAYvHvxF!`u{=Qdq*|3wf&+fDq9iQDotq?5a}Wvq-+%t zsnUDsy-4T;2}h2c&K-eo64YEl@4)7qGf;FkMm2%Zz@PCsSco z1ST7fUt86m&V_f;no$%5Ap7@E%5v_4a1{OWBxLP-qslgRifKUbEarRR?Uy<}kf0Bp zgzIVPvJF29>xPT696uQ*)aJaUb-Ez&I5O^W1iwB1$M>VY+unj5aad1fTJ6WQa)+Jv zwu;XErLiHO??U8rw0lKZz)aW*#(;s`g3&YaHC5cp{=N~OSV+pk-B=8?RC?_QrQRV; z3u-i<05w3w{J%x$(USdDmcmHPW=dB=}J3L@-qxeFHO zl02QtM{9bG_=Y)s$IN$ix_jhiguu+^8l~qH%8N~=+=lH=p_;C+9J!a82BGV!c0j_m zBGaI)nwrFE=qnXkFYI|%E06)YmACEDhu)tLK&`&_r zAR-Gf0H9n+7Dsc2UX6*3nDcUC$sgr8m6qfMzsKIC*|0I_U=nL-Chf|IUB_9KXycZ* zA7Aue6mw*8h$b#N6|0F7r6#Zio3D_R;>njg^2eXm$RIrFn)7-yv8LTz)jw~cTIuhU zEwr0*bWLGdW2NGIt(sxdnwgz|ub^6awjCpRde6oG4yts^@!3fKs%fzyse>Ye7z{me z3RwEaWX}KN%=_=PhUFgwWWB7<7IE4|mC=mEC{EnQG^&-as<-EyK?nmRrqOY+vt5IR zljQpwe@L(x30{|mN(px4i=IbabTQBozy4mfk@Md<^_?}jcrGjD16Q)Sh26x~?o~pp;FDiyE6W6hE{JJ79*T)_{;_Grfs-6m1t`rD z_aa%Frbr`@Y*x7j{IT!MfOi}Ce)8%{3Y!{HOq>gXMW)f0+?e%|un|KjtJoV!~#PKLX)B^N@!Xeah zqvb4`e#G;#qU^(p)7QKg3eD;8Ya;_DKDCc7Oo3R(KSfWrUaYiu7Yrm&awRuiqQb9_ z73m(LM2_u?(w~`)0uiKk@2QtdE!U*wn6BoqMll5U_zZR*WPb8hfF}Fp-ZiZq@chdF6Q{OTuXZ9tu+7x!cY6v(BG-U@ z+bk{yCQSo?oYd_)0OW|ZCnD8w!XPDZKXe!b#w+zLx&--H=o{M5}2pfo_Qg6Y#LLwKX9R$cHKV zcyog7FRD9Px-d;Z@wMRh5z~HD&?5CEg4G5;X4AW&-1(v}TH?y1O*v;BqzJ(D@VQ@X zJ(9QbCS8!&lU{Tck8zo7jhj`F%28vvz4#V$T&qS+9WpRY|LFD`R_so6vpD;f|@Lp12wF`;Sa3V(sQteEA|m+|-xqbVuxXNGP&(^LrSkk&G!J(tC8*(eA(*h;ydqwSf|0T z+e=2{uQ8unIWD(oi{1UwZr%Ybp_P~RU+a*ntNvJO2@J6PE5_z(h#hO$SMDA!8~D-u zDHM2O{Ym+YRbW5-2vyPbDkweRE*dTxEYPEkGEu3yRnD|%Nb~dspcYz2-wjEVdNzMO z$5987Y~X`XFFHmB(#3>Fr}G1N%^~?BWlIFuxrEyUix2<8alO*C_f1MI5+IM;)Jrsn z92IOdVBXH-msDR|Zep|W1<`3uz=DuX?V95h=8_~lrDl(?=ZU!7xo zAQ7Zl)U}YrtdG{%TA*xFx@iM-KjgJ<_5*s@yC|O&Y_+-`E3@>rBc=hwRFrfkeRV5&Qwv3 zY~E&o@%vogSj$ORi=wqR1#pocIkZ^+14p1?g_a2SxnaBbng;#gSK+su9<7bbOxo4* zDBMxNj(nO@U41`l%N{oAf_pg3vO2)@zobT-zYK!fEQ!shD$bXp50c7dWr7V z-$s_dO*VR4@AEO11%sW6lfI)Wi@lx-B?!uvK?DWKI@R}sY@Q*cR9Wb`L*umzqe~I)l(KxBH~c9lZdz|(&RA^{^}O&{|bNKX+EjD#}a2)qMCcN z1D^ovcHxxBy#1yngLiH_dU;pRrS{LFqb6BS14m@;fiGl9oL>en++)Srer)t&?4UU{ z!gDE3k>k##|7tdjwlzQft!%<9=93lf2dut|W`_3-UA=@X3m;Y}eZh*>f|AMl8zrQ9U z-C;qx8@mQibd4Y0G^y@Ns&RL*;~Jlu|-|L zmwj8H@C$shJO`AeKNO|aQ8D|$yQVW&Syc??VQe1amm~dKsmfWgS9vUPm(fT){c_qa zAifq4#$Ef?)GmCvv|jLhzp!u(&)LZ3(~)7~DFw%G^3_FOj#I7MT&5B;edQ$%h#Rl@c99hUp+W4{KVgn)z+BoY zpfx)zTmet!P03k>y<6XKj?Cyn}S|HykNQ_L=1|rdh-c+k(cq>dzKKhr{ z;B)@$j|aA@Ku}3vxSc0IAV{F;WcjlPq)z3cHwBy#wQqHzMF3W`{Quc0qwV2Q!R<6! zpix{{_=~##_Xr8eW~C8%21|)ssy>=6x~Kuc?4e4n>kkeW>Un>iqF-%B)xn}VJ_p}0 zSLdw>IqU6a``0*D^W~h8)bFmR;<0pd(svkmcQ{nG{KMa?v@{^hJN&b)C5FVqg!iknb{ERdV`m8ligH`4M2CGZpNI%hF%yNZsab z6OCPptt*1bO`QIyw3?LCbb{W1?PAK94ZB|chXb&KTziW8!!qps|Ee49zvo^1_q+K- ze5k1wFq<&f!xLkY7~n$sdEV5gC>`LZO^|G-l$@g|UL&@ViT2&OISd1r!WfWIW%>m{ zX+yh)`zC$6QFc7K22_rgRKS`_#6?Bovxe0=Q6Ru`9u59L?1 z4Gav{ZUb&61YpjCOPvm7?3az~X_hv?jC(Z(}m6fVx2hE+&T*C_ah*jhN%aB%OzY)?~ zPyxkueB6j^i{-O#r>2S%P^huN6H+y@I~lFnA02moZx8E`&iW)>d4n{EmT%qRI<`4^ zB#gWg`xNzeX)D2AyPdd*5%dv5L~#WzM}NfrzdQ@zdjGi+|Gix)lP_+ebq{TNM9k$P z6;iB>lh+n*25thDL&Bs@Y*M4d(k1L(b@jdAom?&fHpa z)Ff}$wDVGS(d#|HVLW{u?E})%MnfN$DJIEVcP$volWKqJgN+Y2X_j+ z{LliPzvw3_$=kq|tF1HR?TO>mIN~pC|G8fWF^z7k?J71bh3qt0O!%FT$yEw^$Re>Z z7(zM$@$zoG+vuFj9b5iy?p2M6Szuw=^3d^tk{pFB^+Z{ufI@J!ziR8PAmRm~RBP=7(LT zk`|b!Jb_wI{!Cs+7T)%zKzz39STzdRV*Fl0AS7TaUO6SV>)VVyqw_uTiuKLIYGT!Y2S0PE^vd&5;CR{ZLR%JvE@V6u;)@ zlfBESAjhLx>|UWU9b=dx{5MY+3(c#fFu4|ql138e)%!H_QY|^gdwGYrv67mJqSvQ6 zc&Vt?IsPfT%SR8D+vCu^%EP8a3k;jmFy=}EU{S5!^PTsZO=+TSts07l z`-pr7%-(u4CqqrgvonAPFVo2Yz+)yp>?N^~>p5xwxGuKHY~Ah75w0#vHXsnjV3Tk1 z0C^-Zb8W?^J3c(Q!lPq#?|Y-#6+jg^5GK(17I{=^WMY{*zW`X{;fo-4VAQg){!_M& zbyJT4t4z+Q^_hYvm?nAfz~a0S;^WOWUhY3urn|r3OKvy-AS6f0qO4&xDayq)30lHU zda8Aay2}J%#M1m%dRDwjLfV?nP^FW1aSh*SXMIV6G5wUWKG37h?xOY&c<9<#CcK8a z4MXjzsVrrqk}kZe(w;5FOSk}gpVkA}YZUvIbG_=w%cW)7?@I*k3fy@eN8~LDyDLg6 z<-k@6B;V@S7{Yp9zog4G;E2zD_*l#mtai&1*Zm%6S>%eyuutRO4Bm+a&NYul zn;eK&|}pxt@}VaK$=XY|S#Z9*Q++TJ6?HbzQv3(XNcF|XP>h?RSMY0ypLa3m|X zL}4dzVJkqM(5OM=>wRmTw!JpuFTGU-@E(t#Yb-l1`x{K=*BI`+tJPeJSnDqQPEue& zw7erPX8`M%+&{g=SnUBbZ>lTN;IPjU%EFN|eOHUw!3IL+e;-k7bH-vX=P<}aB~k-P zmUV!Zj1KlW`*1qFOTN{_X&g&)!v_UIzNJ5Wb;!W5-FW3X5MtWsK274R{Lrj1Fgzz~ z66av6Ls~WiYlQ2TMk5(NV3xn8B`x?j`6juYrTWzWyEFALuNT?;C_b8lX4O+5-5$T= z*C#&H!YK<_hd#wH{Hdv`MhAoB$I=a~TjHztIGtnjW)6@T-x6W>=8E!;FB!NZfgPpR z10>C#OcdN$#KU0N05h4o=>Wt+Srjj3yFCvdAR>i=uG9zmWJQU9|Bi}40<5bcHcc{3{`S%VBnfN#&i5QY8u#dKS3K*kMY}w6&?+-jBz583tPsjRF5?(>;jB6*py;R zFi!($_b}CfmkA3KKeaOto22}?SZblvP2R?`)~_d=-U9L$Rduz7N64cD&f>N|R@Hqm zZwoZ0if3lroFf_cgM3V9_vly!ohLyl%JCe)95n@4lIL?n)HG&JCFQpgAM8SuQqCPQ zJ~~ZxUF3IrSYiyui*p;)GKbBk{l45*FG^@9(?vtXkE2j|6{B;Mkk)vL{}^GtiIew1 z?K+C@D?x6(*q@mxlWldV5WdW=QkBPlT5LJVMRvTY4(!{o<_+|14o=Lk3Pk#Omgd%p+4keRo1|5;>IZPSXou;suT*SQ zp)YNIwJfhk#T0*}sLizf@Ra};)M1L+@oV=%N6D3mp8)%C<=EqsM=9ekD#Gg}D06|D zGKYI1foq1Hpb8J6qjy-t^>uNp&VPlwAIFWVH@JPKOpN#jZ6^69(>W7+n4Y#kgIn*WRy|D0;twKe zU8in;9X>J;_y^!AO<1cN>Q~eidV?g&0~UD^jL(XH#GDFX*ErY7`F!*=l@eOifd69W zgRn%{#Kun4{bF9a?#nH_rTQmGv6O9(v<3~T(DcBRk3!bRxpvlnKoV*-Jg`A#6}+YF zs;$)$34x2PZcRoW!^=m0wQ|*+rgB^p0trz&f-`x0mvd1KWssfxu9t_j$HCa-9mL=@ z(txj%hoc1zZBXIe$jTt8n~LEV2fL*lQtJ5oU`@O}OPZ~+TGQj&Ojq{@|4oue{d||@ zd(iiNFS5Ilv-o@tT$;prVt+iVnz-f%Lucf|T$s8}0mjR&e;QWwQ1beO%)F>dtZw)< zq3Wj>GyrSHVkH17-G8L^^Z08i{RKd+PPmHe&A9JVLTEk8#4A+8D6k}F@2 zZE9^?f~;M5EnWqLNUrp)WdA47OLP$)CBrGZM)Qayiz@7}fTWYhicj#~AS8ja5F;is zuD7PeE@^`Ta&;d3^vP@<^7bsv0!$gNuekro3<%60Z|j|y;O$v|OaX9nl-0mdHphj< z>>9oBqh!6prd1`|%9Nv0ihEPyQk49?6YSiF^SH%l&>mi39?$k!{e2S9albcSMx2DH zfQts8KF2v&>T#-8>vwNOJ?w%=xnD?{Yv;{5*%V`ZF zrf=D(Pi_lU2X^jH{H-Rq@8m}7G+@lsVxCnPs!jg`3GyCN^1||8g~Ym3w4#%%`x=nt z_0Ny|H&apm#bE-;F%&}f*Y%%&2xSh19uXI=XswKKk)EC4BOhk9qV%{rO2W@8T+6dH z`_ik~mHI4hpq*fKQ6dLbsM%I3M`Fd0>!EnTpYzubrCyVT1Ut8M{zY!&Jm*vhD)hqy zH;fHXA<8o4{EACUOUDMfdnu=k0yqhh@7|gA697i5^IXUN&vpqb!?(h|&VJl*DVwQv zO62F~e~P+D^@O6g1p*Qg@Sk^oWKW469QC*P;VdlbYc4Gi(X$in>f%8}EZc>_=f?x&{W1j~_G68Md_WetYk8A|w5vd}WO4Qs~HRphz?zWibi4 z0mqye$tcv&zETj%0Px*mL#V%2s7yPc<@~zdGyfYi@S7v0P0F>n2wcizF$&}D^2PB^Bs&)WzHyfVsekSzj zuS*Sg;Y5&#H_FLg0>mh$FeN>x9}7Z=L4vw6*8sp789`xmtU z$DPJOKYvP}M(rf&NKqT5puxxHz$}v}_bENQzM6I_B5cp$x?kAg{n@}Xp<}FFiNDT) z1qPAjJ&GoEos#Uxd@L-ooO{_wb0>{YKwuQ$lGKpbs6GXDuwJB!=-Zh0S8xJy?Jm=Y zvmB%0Ua((0ge5l8LJXjpbd;5q-`L+9N-MX|=?}_u*h`W0P--Y(% z1j%!3Uo+fEsO;Wtk-pO1JPo<5{iqEMp=urwh9}s3a1}9$Su2}(8A$|PDi@+!|8u_b z@{jG)Mm(Ka3H~CdBBFo==E4`>9n-a-t)WH#dxuZ7IU9J1bV#lgsX(#aRnEG$eIfVG zds9;%b47MnigX&N*`6fr2J@`S##lrSLqQ}jj-UsD#5&yixX1B}&Jf9z#r1 z6)EqX63|%8xa(^d+E23Iq)X52w@?|pt%7l*)k1Fqxdu6*#kDv;1vVniCBNP6gOjwEbD$w=wyFtnX?eyUw3 z-_IR}LBE>lx;5T*i{bu(u}l_D(_C+j+!%sqi1*1pDrIl?(j$r&WEaX+eepxAU)BF2 zU(w+%vxJg**DNi&fo2Cp5M%%xTJHK+AKTFe%la>VVfi z_3cTk_geMqeDQogJh;(iz3#)FY(8^P`stiqXO)90MlEW7mUM>gEU@$_&wX$AncgW7 zC~Ppdxj^=9aOFb)1?{_lBR!NCSj`kSy{FKv_~8|Zl$;fPCWh>}L&OIWvk96rQprf$ z=wRaVG-`Kw4XMlHr1`n3ef+1(U-ILK)-Qy%iiYf?kMtn~wrZ@ndt-SNS+63*6qrvN zjW2DWIfkIbEN7w@2THeYA1P1~tXSWwwI|42)4SSea5J)=i{xDAPWl=u>Zm4KI7Obp zXghSoy;jM81(G-yGgU7qy$cJJFzHsy*8brjMev9ve_-Du>v3LPPCqp!>vLRyj<}eW zeOP7|Srf36*v|GnU2CV1(BbZ8xFNB#)&MPrO}EwU)Eytjbv}mjtL+4~;zTVPa-qe3 zI%VyTfVq>ef;&Pf7VRG7R%6YSmCWX*Pgl|}c(B`-rYyeC|9k!}UqjVoAV;_zL}9=5 z(`)OcDu=fiHEy(4WfeflD-C}Ge+Guy!iBknJejlZ&}cDX&Hppd zwqqhZ>hl*EWXIJzT`m_Z&WQ`!GL8Ie%PiJ~r`nmQ8NMseRg^~-@zX^!(i~&5*NW<& z$ay(7U{@N(;70kRY{EXTX|23TFj+u|&*OU)Vqg^*x{)+8?UR<)T!ihbljO)*k72wPTdl3mG->niNIR=E#323y7%8(=M*Mr*0Peet6<1fVIemnnw*~yHzv;M)kKSY|KD;`U{t2!?64|7&DaL;GPlq>^|D`6H+_jz98S)c9Cm^&Bq??#XxRg1J_=# zgYaZQHs46o@n$r}Zx(txJiJ0Z1Eth+hQx$K87RGhX>E z_hlbX=0N>yJuukS$O=?;BOPz_Q@y1}3p(buu$%$ck%={s>@=@0nw zA2sVW+*<)(%pCZcEFbHGJiVB?g06SPD#R_*n=7t}9I1dUk0XrUR86o^ z86l0dq2Z=n?Q$S@5qDt(`I1QA5C=(KhqNh>_)2S+Sp-r3)aTg6ee6t-J_32z@c}ns zrO7#+WSmptlKhKQ~KyNxlGc)cd4KV^l-)SFIb5(nhfu!fur+TkL;i|` z$>nlCPz$*oS@7IUSM=G2>bXo9uh;(fPQ8ki=a~ZYp4I-&0YPu{h+bxcsEY^G zBWwQ%wJLpRJH4sFj8Zs6)_u)9vYOau%OHvwXIwP7Ux<^C;u9(Gf*QtfX|$2OQDe;0 z?63Bi@2+d`0sT!1;y2_>W=xD67{BkjWD{PU2%-OJd^+C>=*f7(Z2|nTziQBe$jFIm0WmWLwoxD-r-LZz4?8-0#`yI(D2S1{Sw8gu-ol zfjRqaSvM75nYVA|TE1lUcKs|AiSWKSG4BKvvXG_=Z_Zksfj;(;?x;_SS$pvj0->E!PC{41!i_ zc9%N$@DSu>zIrssIwZ66oSyv}$q?4|D4I;p7Q_izYNmPvvu=00h#jbgSKed~P(j3JrYRPV~GI`LOvl_ib~cFP-s1HKiV{ zt$QMSuUP)%b8W6?S9ovxuEX&0XdNyV3?{&8b`c^=UmNzwrjYf{XYS{P$&C?)W*a0= zLN+Xh)Qm^Yvn*BLXrLN=gzUd6LLz-~gMDR@Wm7x;BY8G*nBLF%MIL#=EO7ZLyWLE= z$8j)ov?0>CN|cb2=SSZW>^ARebO)$Rd8#WXf{3(M$nH%IC)ArhIjIlu@)VImBc@cy zq2LQ0T~?aj++&f6)!r>e*{4n`*mekRCaL^0bXWnA%)9GgWPNYC%%`M!Bf)m%CV!Z_ zVa~(m5pdYEy%_d*fi=JG zawvY+~?-)NASx*IH8=3+o|C^9U%%yl=;xx zVUKm#JTnnt%iKjj`gGHnF`MV^?MRQ)qbpCE+DNwGyQ@vs<8Ov#1Pno`^#!~%B;}Ae zn=)$JOz)s(p_K9!Q}NZ7b+7U9AzR$MdCxHKC>WRcFspulZ79a*>e2qZpDl&~DsNbZ zb;~7w{~EAsS-rnj_b}n*+lF=0ON3{ofSZXb|u7${m{i%`^IL;ZWf609(d&W=`UZl=7k87kLg&$>RnvtfOIa ztmK|!yr&-&2DMeexdrA56?V9Kafi=nMaQlxBH?N0|1OfopYcNb% zFh`uOX*XDKfPt(m3ytFzDX8)B$U+^z613Q+4c@Nfk?{x#sfGT9CJmJX?-;5IufZuE z`(`@{~3cV9W+%Se#zjkk924V0P}BCP#=WU^qtmc^mH=24?ZI*+0KIE~aIbqXDa z(D&dK8o1-vf}@?SVUCuoTJUN?_JD21_bh5)PI@nfit;BHLQ*z42U85{YJhnzdZLN7JDGgoZr@ zoqo^g=Nq#d&3i9`+Hp`2_9)UjbBo+U{Nl&VLF{G1v5)K+dq8;1G0oh^3n>N*rO$lL zjgY`v+9u>Avp@G~i^~a>?$BOym!<4lD+(-gFo(%+YYX{tOnE7r1G*)^F3Qn$lxTsI zcYu-=$v0IFht?UgdKygjmZnHBa&QSDwI8D5BkYemjxVVou^4;K=G?yEZ+=^?#&v*L zv;XbG(O1y%(2+T)S!EfN)yJlg%#_PL)mj1PT3AsNSsHJfWCnq^E3rh77YX;7#VgOp z149N4z90f=TKk^&ewwh85A?o?(nPxfM-~0@0_Yt=Tz*3Afx-pM%%CNyAz@QxNcg1p zl9lSLj2awYnIwQttr0F2;%7B)zMtT6V6 z0g7tpxRy7__dndl@7r2v0T=ug)zkI*8F#e-riNs>s0)7FG`rZok9BM+?RA?*ru0GB zO>abqqzi2;p+dbWyG;x<#A_d%kw0=U7VPBe%D#ULoZ94pVD4<4Sgs#kY?auT7lV?I zG9a69yO=+EhnBZpp9d~j!}@-eB|!0kU2w9tbU=$t=cxkai8eVf9%U1BNiAX9z;1C( zz-tg*BQGkG@DZX>%EW zbJm7lkw8Ls==8EI76SFE@%x-~%CLa!>vivoUVz7`{v4UKNVox2{*Rh77hrekfoGKU zrp+j(+iBtYYOp}uG7k%>^Y2M2GH#U=m>3-VmbZqat&ujD$wHsCD^Ifnv09bm(Wl@4 zf5ewx zXI(KVTLG>Tz?1a9_L$zQi}X}?TG#CQMuGlbq^DlWqYbHVLf&BPhh$D!3b#5~7VHPT zp3|5hd?Q}6^!~R|RWMXi(Uz713V|c7K4tiz1D5T8F}z5hd%dCs|1{)io-0M}BLC#k z!M4{Xd^uU};(MPEz*VzYH(?>*$9fm0u0p@t$>y=}?d=sFw<|xqwTl8;{yF{Q-|DLi zcy@qG`&rcg`NsJVwei0mvls;iOzUVIK;f>gtnvG?avNKiXJQ&jyL@%CL4&0_88xZ+ z0}P4e&9QmMk5#V`=oT;HF@X)}7kj3O%_+O$3EyR}rKEdYi6W<>G&w8o(!=s&%FBal z?Tz7IulKZOhQpArw`FRcdJft@otTXoGxKzTf6e*gqu>LAKMXbQs)P8ZuGOIqG6M4Z zyB5@_aud7K|9O+uh1}ocP!u>nSX|C??%Ss!oPU>G_4C;;_Z~mIaW1g($-IZ{2o7UZV5oZ;2WdOmDT(Sep(uv?2Xrj;afEp${je|ZZSzoQ$` zQfStFI1C1hiq|guynOM!(J{oH93*w?bj|3`pnk4a2gL){Pdg<$_uKBpUZZq-Xhi<+ zLj7->9D{d&Lxiua55||5i!06~le$ut>lrFE&EXGUUKj~pnPn&MiaE)>FFjm=)!r`C|q!BB;fd((p~%gJmW-E?6aF;{_@cJMx=QlJ%9*VI4U zPk(r(P!L?Tsxuv=F5HRa?Yfp3@?sU}+J3M;+-iUhK0D&EFbbI|AkZ4GUm+$+B#E9b zb|0u%ik!xl<;v}m9TcS6>hlP{URJ+R6noE?BGM9 z!EZTZzS!pip{FKcy8iywL9^JV-(G@IT2e!Zr~oWjQCF-^A(gG`kLSJpi`H+0T<;yc zH;WqW=wHh0G3)8OjS=yh-_ObD7p0HmUhM@9sqace>gBsn$>b`w;a>;2kI!-U%+@aj zeQ?=fHp%wClkL3TJ$q7{60fg0BG#(awTJQt`_r$H^+5|>DZ(W^)!aQY0ofwfRG~qx zEAwX;Y`~&a@9&3shH{r_-OnN|(px7RH{nN0H;DnqHJI*rt9C{3j}Pc!4dbktbC0E> z3;2Cp$~hfrsVv3uRq>sDT?^J!cO@>&D|M`sbe?VAvCmaLnCb{tF&vm1rp*L8 zz)kC&?M^kw8N#f_=UV)$o5%h-O$9r3Flp4;d-8q#C03fw2~u>>?7G_&^nTCWB7Ctr z34VhHA67cprBX^9ZdlPW?fP_-NEsweB?d6}(q^&dW_(zMVhX8?XmH-7MY_Rk_XSe> zJ2nFq;0W^C{2y7r`^81{@R+p|MY6g+tbE=F0FiKPCXU#(I7<8pI*L@0@CHEVS zUwE@)A(lN!6dJ@h09{m1kMvWSj-d}m<9$H=X-ut(epsRrz#Wk0S_HOZLk>IGWhRrn z&bf(ws{+PC9nGj`EiMGlGj3dWc*PiQz2gbK(q;7K!QsM_CWg8tIB6(vekH7^J=m=p z7Ym>u0b{V2dkf)EkI(`AQA9H;v%ng;0c+dU`Ci4u%5t}PU^>Y4$V@OkRwE^~yW{c! z>uA}}7O~Vj73q%@2p?#{{*Npq)i1#vobZEao2%_0dj(u6KX86~X1_cDlqZ|o-t4$>8v4;8gXuJQ0v%IYemouz zLN@I#j|y?XE89t%<@a|8U0zT%uQsXa-CKH#uj>q#A;czpTZutIx%2mwN@|8hUAN_h zdN!y8SA{H79|@wh03k^2)R`x-A<%JbBS|7b2U{CMc;xPEuhB-)UEb~17b1T}fqmEqsFPIM-%F5YB;@1jA)x*qpX^PwQUI=$WeS`P_l}w1>^r6{|fNHJddDKO^#6d&mHEG9BfRk?BoI{bDVsk?l># zZj)g4{;O8Q(nwu)cQF?_zMh*mQI7*6P23E=4nxb6B359#dwa|swD;>@&hh85h2m4k z4<;w!MmMk9SA2 z{S27=gYcS5H-VtR#yPlLC_Duu>sYI;zlI39-XO?nH^a+0L@w=JM(8p0do%(|0erht z-#FP8+V?QP@n>c_3eycUl}*s6JMf1R^~Vpw($JkM>7#*vZ+a}yYI(z~uxlScK>kI% zqN|!ad~2+`Fn=7^!)qv{32v#ZS8ZxKw{&1;peyf)?yb78#4~r4q%qOfJnFgAJBA4i zE+!_-7c)&bVAlk~-TgOagGgJp<0?}0Q=7#lC5hhoHpgFO2)-cTF!uJWXNaR~&+*~r z#v$X$NPnXA45ii5vee7Tu85u3?=l|x$>X~r0$fc));8>>AA)s&ZaVm>;-QJ-os2Zz zDPPx3$~O*JXW5=Lt|YGm z)DG-uXhMHGvcf{{o|E|&`o>$Y30;PT{e(S-smhrLC1?T}{b+whOeVFp{>8f6M`GDwz?~=4I`&25)wq4~3u6j{#J`{PIEB7X2str>LkulBhc7CtCT7uv0#D000 z6i)p%gwj6eDC4im@)_<;9C1(K9V`WurnMWZP;8MI%P_s!Jv+#*SsggM(i4cPUQ|_4 z85vGONQ@V$`L#iBUL_%h)`Eht74%NfMw}m3YqTa3-&2GWBUu~>c3jdDFI^u?Ll;K~Sl&eLC^oNncM&!o@b8ZIt0{n(Y3qU4%C%#5v^iCXB29ZHZ#MMA(Mc*cr$PQEWAoZ5DN_ z{!XLTQ-`?_0Fm)XOImKek3N{(2Zl5pawd%KWNj~ zNHJ-1-%?Ts*vD$U7iqw14qlszA@{@OIEP@e33h#kcBp*Z)oTCIJ*^l?TI6*gt~3Vo zf947zIt*(rtUAOl|J+(&V9;YQMmjy2i8NgFa#7*>{Ou0sfPy@`)Ut~kFPryoa2 zDo{KqQ+b)W5~7xD!?TR@v6_{teAevf$Df0B z9$eYgH9a3g_g>8&q#*fBShCP_qWO7E*j(bL3F|hyX2iB3Cuy5u z7j@3~d%ZGq(Wn%!O#=H-x;M`%c~DH%&HU6YEzJ%oF#R`$aL&OO4|QWbYlAhBy!jF# zib->mp>RjvMVt9W>sK?r^4Y?pDUm#bT%#qE>@HyV(8BiM2T{!oNHV{afYzprr(%!1 zOe%qjligK5_3CQRa+wls&1zf)g^K4 z*XF;$9g4s;;Ki&f1WE^`1056-BhJAZvRX1oOy+&a>|AV-YRVj27kq53-B;uOo&6o` zG2DBhJQ(w2JqO0II-Omh2XlY9vW5`f*=(5K3bN9!zxRE2t3JpXL%!h#YZd_cOWFkW zi18P&Qj;)#^j)lK!!Y2(k8*S_okSNEP;;UJR9%_CB(+!05B>R>oz+YH3GR>N7jA&0 z)w+j$I`IPws8Mf8SM~BZ>$CkpsoblMO>%C_?NVV`zzchqdTNn-KjVUZxd0hSthTJ^ zoAk%%(}#Iuo$D|icDB1I{bm+tyRaa3rI22h?5!=i&n%ZI(K8*Y9J<*ghKr%mdMZ*XzppB$fl+_l}Bs9M~#{r!sh zL6`)LX#EgCW~dujht_P}^ZmaLHt>wPk#^?Lo649SK#1cmZe9 zYVqd2dK7V$5fq~kmki-z-TAf#VAW31Sl9|H?+kAp6w^R&f-5T+&mFLmL5tE$S|~CF zxS}w8*M%(&(&-M2p1~*I)GylDq(4=V)cB{XQfXA!zVGddn%RW#)VHkn4H#AW)kp)NqN2holgqJ^c+w;RODnjo=@f%(N zwZyuXWjwlS8zwV+m*lh=_`%~$ z%BM8h!7>sl`VJilOmOS1mn|c~+~y=La|&1u(I=0;nH=hk$fq4SEW0K1{lLrwy0eQk z`Y2*m#;aA()H&p}epaFEw`IP~Wf%V=P&TpfVg57kPx2gxvR?2t+t*HEsCCbqd`d*2Lh^ywz3!k4VvH0TdS2hV;IfQhvIX03yT+ZLA0&_An6S$S3! zTt-BhXLtz#GYrS=N2t#%#H{hmTV;S&;K#i5QNM#@{%?}rSQSUl3u1kq-N(j)m4!&C z)X%I`DGv+uZhb>iq22rDrvX@DsDdQiY~v*(L_QlHR66NG^j@%5Eds*&o{og6#Mpss zMMa0~;N83#exJ5SPK=3u!acUzO`iCo%gb57u1mX~q7T2W0^aB%HF&B+(Z-X2oQPI+ z>X}arkO!_s`qOvHX55>FxXO$THt#+Gdo0h2OG?roK>&l5V_Ge7Vv4OS`ae;W_=AL4o`be7VpP(}G5zy|hlbiP9#ryL731pUA+(%bYF zi!@sHy7iB52OKNPq?wYPRpDRnnD7lwsTWz)-_O)S8E)P!e9m&@>O-Lm9m*}5!xLi( zX(h{Elg&^$cwS_~p1Xc-pTgN}!fN@&Xmpp7d_;BYaXdz$6L$_62;e zK*a*(EsOwd`I0h1DBT198fv@@ylCW^1q53M4uAG;RE8^ktu9zNV`!9*MQ%Py3_K{a z{4TFd3hjT3S2N>|swovL@#(<4M*iu3SwGaCkp;1b4Zhyi)yr`q(R*T#46z7aCm;!} z#}=E~RH3SIu>)=t;^#fE6_*?#Pz;NuVi1P_+4X)gys4j}-qM>p2#?mEN`q=YG2R zl04^}wX@3J>$mn^d!4<@=kx)C!Il*kCTfE})~{gR!2m*j!!Ok?tt*fG9{Wb;M6t56 zAm{~jYK|nE2TLkfn*VrajC0CNxiC~ffxZAgxHtH4Y-e#ja8k%*IgSV{4{OX2gsU@A zyg_|&dv;66;MZdU3d+R3{(7)*eZ?vJ$*nhl8C)T|rOq2V2P(qnZ4Cxhuf91fc3%it ziIw5G^#Gy~FM%LP)!(5AUMqjNGC*Vniw52LWGSWU% zHcBWAZu>gcC+q5$$*Gc#v#JA@f6rcTcr)(3|4$RWY6{Y1^2P*4AI`{;Uj)kOv~C>A z)McrPdNSK)rj{q<5+3wsFMU!^T4~P1v>M%i4g#efQNqxGWibbXFkmV4g6Lbphn(2a ze}XqSYfyuuPevdY0(7A2S2HViw&}dxy9Lzw(;6qA?VV^Z*I;}rK$)K!^e~?4UEc|} z(-I!t+oS{>I2YUd6@8n-nr*r`eQ%nH4M!Yw)tHi_`!r? z=Q9WS$h_xPAdR4&ZkAf3EXXEr@5Fh^er=;t>|a(mYt3V6t8%5=LAXcA;E>d(TiU8c9mx+5`Lzj3eC2GQeI z5x}m)2|!$1KwKw4T^bWsnw#uuS2_P=SKHo157FBRdZ`2CS^S+({rJ0cR+<+|v0`a? zKfAfr+sC0^=IOV8aBEj;agwj?_$$c&88Vog@^s7jDS+F7Od5B+%C+H^sU_`P*k`$w zG3<(!CCxXu@X)JalGt{JKba|X%3foGcHQFsjrf{^*cfP_A?tbH9(SWE{d*fohI2Ux zyHjercvtN0QA=P?)siaqr{w{Ow~1&RyJDSzP#P$M+mU|s zm>+aguQa6E8~krPaD*I72Z7=NowP4H0BzZ5>;6zBsJN#Se zUZ@C&wV(8}65~IfVf^px*Z(z9={3ZNE>rOgh^{GNBMYZ`3H&Q0or~g^ENEx~sm>FV zD{mn^+2~az-e=~r`bFcpw{7#DH7u;K`y6_CCwV_`dJU+38Gz1{*ZRrKzO^w}zf%P> zcbJC_+z4T`g}5oAS#HOl0cFiVF#@39{gd<1)^usqGS4t#eezUY*z9uu3KPZWJrO5Z z+Yzbq3AYGsj#^F9#9+$6s7_;e|b1YV3J-;%TrqTNQmMsFomB|FV<;O1@?6)ec+aH z;pY2CZ4N~(fj~+&mVcT$?*h_Y?GMB%5iQM`XndLPqJ<1VMDN$02F_s7-!=jWoT9x+ zqBW8zfDzqaBF+VStpc-TjaMo{mV;3nBDlu~(i|R&N8RQWP-&~Z2N$#zKo>gVj_I1X_~c}rcAo;M{ZGKT zUurb-+AoUP=sRU18d@J!npIf%=-w={;8HgrMZ;=hiT?x~hnFKTaxGo4(!d1`S6~sn zyykm?#H7^St!K~qm=eM4T&KW^_0GBQ)-7IAw_0|9>@36&IigocFTmS(eShjQ?B8*( z)cwLAXEW!J+u9F_uVjF?gW^&E!}zDG3k4Ru4ru~#o{VeYXF16Q;J1qBN000%W-c6n z1glv3PEe*Ti@&(ImIf>Wun&RXz}-4&{mFAq6Txc)w|-mp7ql;5aKD;HN3k->p|#=1elr{2@Z- zHB5(1N(DHg_;8fEy%Hpi=2G4-utur!w5rCBYzccu*okf&KO#2t0pjC%xdots#*KqZ zlLIVtb~9KoS&^4VSZAG?lnNU1K=DQv3sexsZR(PW1jDwd@Ae6YH}_(&wm+N?(7R^3 zKBDibj<|Z9H;sY(_4C$#r&62L`snByKE|^Ty=vz~%t3brxa?8lj_9LPlpAMOVprB* z6_F#Kq!VX7+SI_boscV-LED&`S0^m3eWvj`(y>-=pVfwxDGizTbKM)+&snwg*uZ?e zw-X*-W+Xi3 z9vJf@cLh0ml9eboV>Omf)wxy&#NbDu>#bJi0v8TTheOdQm)=EZe$P8thz$xNzY*VV zOGe{_yr}Yx7_&Uh8njql1`2YT9yj3p#ji|-_0*gX>t|N(&!ITA>W`%nt4W8edgF~N z0_Y!-smgV`!9!G|lJ{GP5gl1-uIj_&cLggh=Ni$jdSn?`$NSYIY-GV`nm6GqoYN^l?oHpd#7;~ciztj zhZr4j3UbQUtGJYn@b7QB;|kex)l;dKduh&_D;s{uwlU=55W&xVTHaIkTUo-Wi`ilZ0fU$(lS~~_my$f>%rDm$Ngdxm;APC;*SiLg2|J@@nNAjzT`}i93sk;2+3td;n5_zHHGNbhUA?7W4G`pj-6Qbs6^@s&7K;G@Z0Ck8+nk7`iRaX@6JoB~v7_IQA~VE0@x82TFAU z+wE$iK5Jv*_2fSIcF=(f-pG{bp;V^x9`%nmB^I6xtz11G6E z3NI+a>{8;dgG1QEzHjl>lI+~(G3X3$%s(b5=o%)X;8lyE+o z;BjjZQQ;2iBBW@k2@2qkDx{1X6(cNmvBp@8Z@jt!jQy5CCox45;{yXwx z+uk7W*6&DaYJuQO0;Cd;Zb}xM6J1DIg#}Q@!6HNqj~MK#Uvu6>JFtcTf52bPZwHfy z8d{f@gY=;jwJzL8oVmZB9kSZ>jXhuMJ4QS}8E)GXXdA01AdWb`SCTkAq5wf#XXMnC zjC%t(8GjtA+9$r!_!boBIsxnv(r^KskvR5j3oAJ#o*5OWRJE?8-hQMDapf!`%WK?4s) z_}XiR<=5&IgFcYQ8q`KcZT%9&0Yi;fptM$?qFEe8I9fF^KxwY=;ms(W z!2r83de6Sdl6-A5l}E+Q73dW|(wYz^?DQ0cpuS&2G``&e@=6xVfR16d4FdPm!TUj( zy8F9SG;$^8+30oYEMPwPqM#y6K-4CxS%RxtDPSt9Ql>{`xE|yev&M@$a_Xhmb#S<5 zU-oM7$1B*~2f^f;pdh?X{gPUI)cm)HD?>9j-LDu}bDN}^8&07TtEKXA^u;CKRr#9y zJ3ESB8``c9t-N4;Qr|E&S*uv%y|d=5G;^O3grk^ADM`@FqpCS=heZv6QKLC-7%)os z$Hh>PthqPx(58}Ozm=IkCdH9k&wGT~R+iDT1OnDMYKqwXtDk)x^Z! zPmS+_0v+36_Sf{e#ML`svarWzYFvYOH1+-n+&X9m-`fe%uvq;w3}OFzKKxL&7ngtG z0F|^adzyD!h`(^;+KWBHwjzNBaI!}z&@CVrxs z%=_73%Zb%Hl%eZlMAM3lNMotqacyh%b9i2_t=e4g4~>E8B{eloN=CIBTSgb}J2f2{ zuV4YS_8A?6S%$^TA*aot8dEgVt7oCR-dj3iVPN?bz(r9%ro4JY&*aZ2_CB$Fm=LXg z3(cnu>6Bm*_2E)$q#bEYsN^7H@WlThfTy4X2lf~_MU6zd4ryD2? z%uG#9okHxH90mLJVJBxLB%!skK$40`?1vMKeAdnsW<6^Wji;gi#pXxl-wjGKjr+E~ z2>AGXwBx`ekrDOJTtEn<&cPQ3ldq(uB_51dyGnti_gz$N#?9pA287Hp2}dw^_(8nX z*PGReH6p+sSL=`aRPMX>FC6c_FVM}}P|n??uoyxHjtpo%^LeMOQr8xa_g$)tS?H17 z$f3{RLayfDmpR56r)O4xF)>&-TJj*R-p{-!Oe~HU^)k|Xy{`&xOY41BWAox~4G#j2 ztv>KNOpTc80>jS7ZbnMQkDg$yCVR{9P+q5!RZ8ncy{trHHZvv!zkdDx-<#V-HI5&E z+(5p`%5&^|JpW(0x@d2Ym>GVs`}~=e>#oHP;;0{FC$noJKeKde<3GzE_rDWgiktdd z!{phNTJo8Mq=Lhnn^kS}AI@*Jak+cldwsve=9W~iqLT?SuZATS`u`aDI)@6i`1L3z z>V*m3aA%XWeWSv^4-Kj#Kw-V&IL%l;Ru`ilaw#RJ#mhVESavhOD-F>$AGUj{ajNXe z7cnpG34h6Aw@WPaf`f!|Sudh?ZC*W3IhkO8cWcz=Rmm?R7CWTRO@AZn-ll*KdH|Zg zJa2;kEHN))r%1m5E1$^6X{#pA0H~#@I{B>dW^Mq5ZeZcFsaT_v2``ejuAJKSr8t#1 z%V<(_%chZmCBYNEOzX!+B@h^Btyh7EP#fIZ1Zw+9 z&7_;fAun@}&}_nII~U2{2X`vO&{w-OZ-`|AJ)D=*C7k!D`=sFa?B>GjKM1P4-zDb% z-kn#q=AcO5xKK&Kt5n!xoM?=k82N*^wTD49F#DkF2%UnKC};%#@?AIidgU?Q!n!)$ z7`%I`Oa#o5_bCHWMQX}@blcr22E7^2Bg}v%6u` z=kWqouq(9MRxE;fv4+Qv4T%Ny7DQO1Syruj+noNwsIBMaJ?hj?D@`t6sk`~5mMPgm zx7}ewc|o{Wtwo6br)qYv6f=cf%WDfa!%`-7w{eEywC7GvPz0;mf|?p{OK~7CttDcd zFa_QgLD%X>lJ+@4Bj9u(A!+hvZaBE7D4trdu|m-BrTujSC49J7b<_j-AsKcXh`RP2 ziqYKAakh@&Tz_s$Tdgl8bq&@5hvb#?ZT7l+gZ%L=&NWXNtp(#pY^V$MqVPtu5)GNj zFH~Q4M$@6dk2$sn^$S{yOlbJsv{?rWBeKh3#mExv(yHD}Bc=Xh9KhL`)t52>%e z$nrU=F6B+>kva|-S2ou{>{hglNjcCDZO|Ck?_O@INMcbrn3^b~r(d=K;8|CPkk2#~ z34$FqV_`4lhoPB3RO3y{tJTx>voM2cAr(2RFv_q4#%!tF+TGoKlw%_wq2o91>bxUB zt(S1DB`iVdS~ZXz!WXu({{Fqz^%ruHm^lXnps9UExaMm3k%8=NMt~jA!So&teb5gi zge|R(hs_ISDP+ivXnXttt;9K&&ul=DZ*R1wjHfxyGlqhj!!ul0l zO~6u}!2$5HnA=QlXbu}F7JDX##{FI1U3ZEm-hc1*=;3iJrHrEkw=YYQQ~*SndRKkr^G3Jaj5TTs`wg;-j%BfCuqxm)WiD9Id5P--odonhMD#(jPrX*(#(*_62V z-*tzQLqG_Qa*q(mJS8V9RE)7cG^xs6HL8gS%f_p?0aKsSL6h`B8Bus;W6`l@Ja^XZ zea#Ut03=u4YH)>`ZuL6YXyV~I4CX`*ycy?*h@g|lB2uW-_+VAt+Z1iSj(ZtW=%JQKmiRi(}OcJcI&nrn;PTm55GB9 z@sOx0_G;|`*4~Hg_5lGs@1t%@=cgk{2rhBdy&q>`Dl3*JCDTg6PFgk5b!ols*pn}o zj;WKB@QNQevgjoHhy2W;{wM1z{`1@*chDo)=s{@2 zxvN|at7&4cGz(8c7K*15z-PaWH*yDZ8X?r-WuTRpkdx{|i)^$L_m zVUw3EZsXEAx6v(@{ap;po+0<O*0qLT*KG=$;DqnUU#dp|)N$&D9m{Te9y15-O9SNDiKbFm0WRiN^8New zgN+9Q9>VWx%27mqV!@2_3Vxby3S=Z52u z1mm6pD)CBLM&InULPu+OG4@d|><_sRsaY%Wydci)_XOs>^GHKQ-}5GNzH5$Mq_=TUX*3`$@LCF_f``J(M}tB7$`T zn<~r3Ha3rq`eeG=*9&}NW#-?9sbC3`l=ORLH#b z*Gtl?87V7tOd!L;wfg=sw(OF@hV|Zv^RR7c(aiO^Vn5A4=Z?;}Qjv0F+%{j^ss)|w z0b8AINvAJfOINp_I{!C!VoIgAG zRWZ}Yc@A2yucU2=SsR%{uZ8xn>j5SLjmv<$JUgW3n?vWoJwYwA_?glZSqVF)bXbqz z9y(k*vyY5`+a9Hbbt87VslO5NIreTj1;&!GsRcuQ<}KMa&lVQ`(SGW!Pv59p{NuvB zZw{v;HF(i~>~!6^AeYpH8bn#Dp3K@ENbl;@q4c_Q6(>^5u_ITbYEuGJ-cB6tHHgVq zP9kOZ7mPN4yjIjBd}`^)%-lHaV99X>k0|qn9sF0z6{CP;+`aoJpgvW~P*p;=>6bh%<9b5F0 z7l+iG%8BnBNw<)J6U~i#Z^B2jeIe>6dQ-dE^|@owV8eztG0Tx3Z6$&)z)xKFF@F=^?6TGMBRmmI9;UcXs8`@n;+_(UFFZ34WNZSK=ya7AD3h;>*P zer8RfX$iiGOtjwxovpmGoG(4sn0?#8oh@KvS-^E7UB;obaD>*|!3|l3 zZi!ME{Csxy?6a`Q!HT3SYOph{&-3#wwW9Mrr$+@EZ`{9s==w6PQTFe)ed)Ck5#&l2 z5=yyY+@d*U)ZBQa_b?5IJ}L0@P3eEN1lFC=WPBX&UmZ24!Ud;7rcBy)i`50U))4M; z^F(H(PAEzCFsnPyuYXtO6l&312w<3`R#Y{FzSlh}N2ReTz5B9x z)lAu6?!7oh2zA7AM`D>&Bw=p!P&yB?-^P6EdO2>iALtTq*@+b5Q~`rISm*Bi2O%Oh z#Fxrk`)&!@32qrao#cV;cN|};7X#}j81fcVf*_4=T|O&jXHaR~3H5!wSga!640BzO zn8)ua z$oTs0G`ldA0aTLi18h>r%_dx+vg*e@s*B7FWp`%9vu~ z%cm=X>&h%o$c4-E)()!I)I{9*Zloxs&R;c&eKBEe?B&6_nez1(ymj09kl=lWDvV>qlKZJ=_EIjM}F!DN=YEK z8SF}f=P7(L;CLDvKd6z7z!}Q=Y2}{eRCo|!m(=0|^mb$oXGGaiPQapK%@M?X->uBwZ_l5(Hitq!(wanM*XRHNhXd#>PW0_7Z2WB zGTSGtZ(-sH2_4I3yWd5{biToC)G-Rrx1NlrjH7ZC?rMm&Bb|KZwDrr*z7fS6{rfX+ zj@wS$+<4+XZFxKiSFN@=8TsV>dxHA&IHc8K52?<1Z(ulc8w{AdW?5(hw*OJq=3#-` zj>w{S_Q@wpzUwa>qV|S}ETypeterRYFjK9_v@6!9#UV=j+rsW;1p|($a6}r?GbZ{f z-4<@#g^pUDTVXPVSj8VJ4Kbz~s7J!pW>F?{jc&g{Ct{h1Q`Wqej<&B5A+K#gnz`1$ zrY0fg{ge6j3Gp{5?TO;QKYpAG66d)0XKwNIz0Fk-kk!SXxv|n;e$?`wJkYT@X9N`< n{aMlK{NJtpZzKN^oh*og=!6f+SeF#Bkt{FT{aJaz>)!tXz%XaA literal 0 HcmV?d00001 diff --git a/qtfs/doc/figures/offload-arch.png b/qtfs/doc/figures/offload-arch.png new file mode 100644 index 0000000000000000000000000000000000000000..944900b42c13091e4ec40c6d51dc3c95088aa1b8 GIT binary patch literal 39249 zcmce81yI#p*yjZV3PmMBh;#`^OLuqg zbKm#-W_Q1x*`3+hSq5jA>y7{aobx>ASLX&jQFWl~ET-&C3d#e=PQEKzUtxdn>ajaiAUmMx9btU8}km7trBOnp9 z>?r}xU{IB z>)&!|ClIK=(p;Lw5(D9f0f)ql03Uwdp@^z`z<^$HC`EE4fJ82^?D5 zgm(*Fi2>_l`UdF#hIP?4pnpVe?FTzk@FQUt6mk?^1k2olABmf z(WFwSOCZGayBc46<+%XIMA9vMg_xbkDT`F_rcVYGc#1ji{oh~r|8(=pZ`VglZ(#X1 zuCAV}B4u8@pcx$DB+6~uaj>ZdvbBCFE%G@86G}-Y-&pH=I;J{*WJy{Vfam@ZgzII7(6B|BZG#} zDA2uDQQr~A)^U11OLfV&JS;A0j#I*D=37M5eyk*VUb4Xh*ANjEmEK-sY-&2QoRxAA zFZ?Y(U&_?1ju(aZ0Ro=w(!@-e}yZy7`l7zSq6hAi{9r68aZM8dC(;_4! zL?EiFt6vn4%r$7qRVtdB=hP^=yPtk2O$z)?Vy4=QZXyxATdEZc_XeMmmYsc-HBOEj zSzS%mBYG(m%c>VF5rF$`VSa8dH7yN&Vqzj^p6TJkqiJzTyB=eXFZuZjcPz054&&wN zg-+LscQ&zc$*)?F1^wd18YOf}Pf!0oVrl=q#&y?p@9^-YhX?uj`FZxYZ)nsLIW?CT zr}Lc$UnBzNQfpFE5h)%!*Wj}I`ufmv2d^X86T9q*UU_)vP7)!0|Nfo55Z|>B?=d1N zNwa&l+tVQ`nAGjl!KV40`g0de8ug>zuiq_fY~uXa{QS^sPF8YdpFBZBWMpI*cEqy$ zsd42*F#fSNH%CX*98S9L@ObS+rBKt*pdr4Sc9D{cc#x3sT7Pbfg8_WR$jF#uA0g&- zc{-IlbU*DERLhI3&N`vvxu~iR2LA=#DUSgQFF856B{FGALJ1-kZNlI7Nl8gLZ>;R> z>?#r@CDGX7VpL5H5*3{F@nM%uhKFO<)_(c&#i!$_3gd}n<4ftpuabgk zt+1RFKl9MX3?-Ohv8OADtH!=wGBJqWjbqkI zOMtEv2(u})|65gGi!gUECo$i3e^8?ks_%KIv$^?09-GMz8ynUWmSshHQ|BAb54KlO z8^GfZcFJZKF)V-a3s;UNYaI(b(vcJ`A*ZX5TtYe{(S~U#vlhy z_y|+(*s!DCfdK;cPl|GKy9u$MJ_XXrguTtxdX$--j^R{2vB}}q+H1*^{_Pv1Zdbyc zoSdBR=7SIUk`(_~mdWx4z*VXnrBe8`&R%mH2@AvhJv4YS+7lBhKD zwK2n6{cSu~b!6q_B%VLNQ@5XQ(>S|rxH^;%k2FDN*5pZpOAUcRfftlpPh5u=go|E~ zz!caDB*`i%1+>mK<~+}hh>gX{9c(X(e4U+>vy;A90V>b1n^Br-yiZ1Xd?1|TxpRmWa_7ORwMRO0s_`!HFu(8F} zhgjxmdKKLr6-%>Pst?24pMY)=ZMV7c?ebZzI2Ty7Q^a3z9hyo z3O0Od#jiwOMKk=pM8vG^39D;smNkUTJ81IZQ{`;Gnrsm_X zP%B^gb5WE>`~8|szEd9dXggxe09(lJbCX@)?H=EcNs&ZE2r8y#V~{hK%QdEQT+JZTVQ92pt;)!8{0z=f}j7%R64cOYO;v~>}s zr44?VnVAU<^iQb;F~V@TKrd(Nm7qQ|K?|;Fi9g{|tcQ`&3{L4io||}h*h243i(TJq zOAt-~6e@JtHtK}N?|m|mGPewyre>iL=Jo5>dxwUI8j0QyH#8s+e+QIA2hT;u*8ddx zH#&XVCQ;31@5NPN=N_HYNtG$w?OPpKT(w&zbT81YL`PIOEPvcM0~i%xEL!cf{^P4c z;xkoMf7syO8YfC>!*#mM1U-oO_z{{%bM)uWv?5(Y)#qaiLxQmtHVsFw4}6d9ph`*SXijfrsL#% z)88u8|3^8T!3l-!gN)om;QTbC?^&l z5b`rMwJQjC3n_r;Jv?)OQ%g$J9L39Ff|2vlr`yhUBE%sh7sV*_n=;`wrX-2^EzL)DR-|%NOy|HOFDS09Zu2 z6}EsGO*)A9%?iGKYXoG}Y07#1#*ODxW$m4u2+e%;&mx}M8#O3IG_y8R3~$o7 z2!m3wcK*)0kiYho>|9*;*rE~=9%2W;+S_Srhd0AGz%T>Ep`!A!CLd74bM{IYGRw&t ztz-jL)pYmmyVp7GhEbp&PxS2jnH%8OGpwF8N*!iVcJ|b&?|ZFNr&naf&(owr=8mEF z>4gK?FIjo3)3Oh`-YTRI!2y`C7%&rSb|2H&Lu2IQV+T*RNPf zn5qw9?*tfourU$R5<*M|^K*gVAq}-(ZoA1l{&&~EzM=w!r1aMb>JDo+X#fDhZYwJ* zyHzd!_%S-A*U<}vN>b(3m3;MFa=xsLjK8N-mv!cz#UmeJ)cH>@Z7vUNqJ(bu(deeX zU`YAvzL?~U{;7fwEhFS%?V-j9{w4|^vnA6radSVv^I`b+@4sEVQ_Fe!HJ#~Xs*E=UzMyvZS_l!tn5~UsnO4J1hoS-h2Ys!XcdC@9ELO-kxN|pk$#q4%5k~1@fEuS2V+o z@k#_Ds_pGSmeMRN9nC5SMi}!N>vFuCH_;-@3&%!B)f#Ktj2}OK>~|(!51_6>m$jt4 z{H38GqO!8G!gkv4v+rI^DDK>Z^AslapyFw6A0-7v3p5|#1>DCZX9J?=1PDfp2l_fT zHf&yp)l+DQg4zS^<39{zB)O&Cf=+8!JDiA_HUDxekn7wxp=Q`%e(R_IE%4)@xss*|H+iEG#TJuI;lOGJZ;-lP$r&TuffQ za_^d5qT8ku6x3l2M|For!w3r_6f_aLuK!d{?)eB4zDw{MbqNU!94vp0%7AA8d;zY* zV06s788}Zk?wrAjCc(jiC8k(~Fv#dTx4DF(2)AEx7LW=r>_C9o51d7?^6Uwa!_dd7ST4hkP!J7b?I`{ zlSu9m6Xz~4hTEiKk4D(Q0PM5`UoE*3&_vVohL0TIoJQgO)wfdW+7{iP`4K*E-(ERe z;H`%m@fBi&HOkF;uriq2H#GDfYDgDY9du>5UPOkN-j!^i4l#$h(5;-m0EGuG>9X6! ziy{DXi;Hltn4zJe_JYu1bIe0TjVPc-P}k#}I~WWS4r0ZSGNSuFEoWRpLa@Qv46%KD zTe7hGcX+j{Ac~WKj#K!mhZ50HW;JHJ+<)J4qUsJ*ru`F1AuRJgd4BY6HFOhYR$B8C zGH4b~MIWTxoQV*>&Y3)vo$e()O=WJEk_;{Dn4h_nxoS(kj)2|?yC(ucM@N@qsB?OY z$n^R?Tyoppq2y%B=il>%>csv&u>B~qj`@M(PubY|_p$Yz&6?fY=fJG3Ju^lu4-%|7 z;sHD?aB`jk1wP4}mzR~jYMwV4O2%shJBxUaYMGSH>F6*u$HM6Pbs$cWY~&>!aLw1e z404^4CksE^DOB4@yGbg+&$AcHqO%3X*p{hOxR75ysWNC5qVk{?x5|0aHA}airJcVd zi3T{2$`06E-fuRGeIjA5EOjA z8xwDBg5}KfMj*+H>RLh9kB7x0zRC8o1SO^GKofvci90!!d9>bVVNtlKo@=4T?;?EE zFZVAs6q^5F@$_l%II*~p*|muFAdiXpPhs&D7S!*b5w_u*w8bUgAZGpbQZHm4BPQyK z1&BGo7`d6vQV*Wl0tFZsRK>NVTrVH(P7$Rlvx$>L9(W%CO#lLFi?Pif}lkhP5*fJV(ituurH!tgz9Eo6r~u#-TA`p zn3KHE9+BWYn(zvy92I;==r%3#(@1{pS&UnAjwTzagyLiOLzCC3X zujys7x!rXCI%PPvu&}VW!5%PG<)!bO#t3p5_Y=M_CzrW}1vd)LN4jbD@-v{^B!x+a&KbYA6hRe4W_;D>p(BE;$*LiV*nYh0s;R92;!{Wq2!rv%xeW z+hU5Qji^+(>+V9R%~YMQqyGGMBd=ovX6-*Q~X-BtF zFR74vD#;ZEkN@h&|3|TA`2VJ5hd5w7UI)SsL|PVf3I)zOB}xgnhP367YOzFy(U!4y zI1M3p;Su3~^kY+5A$_yfkA_j`9w|Rw^3aMhYA7 zs4TB-Z1D^x z7XI!u3^57^{PE;TP|q>UJCuEVBN0A+OFgXo`V<3IWfhf~Lk~)GV0ggg6AI7q-`&8l zN+Vg=^keI7p+$)-#DWrveUWrBv=eVXEpEHePFz6^!}k}`i42PK3ky4b5oSd2+Q-`3 zq@d1tOt!2nEk(pVk(2WuSjMUNP_GgRJsf!+`r*TooHYotirC4SW}Qc6%_KYv#mwD42h!fp@I46kK6}ZmU2+wu;f}`Z*Npb|_RqAnSGa|N zKmd&+p`f^F!bvZ6#cPI2*Hs1oatWFPv_q6WRG9?(E*CFv)9f}i=m0QBJDXBcQajh~ zQ}QP%4nFBOEpFR|h8`Ioe+_Obpvm{-#4Tm)TW)St`6D5r4<#jB$Iitgd5OO9ERqMm zhTkPlVm^_P30RR()V;#U78pI2)E1^lND~?^XE5OS*Tisw+)(xB-0(y+P5x`j*E>4;Fh#DJ z19pJuxNR39{l#~Yk@)<-)RR{e6A~gmeacQyhsM&~t%y|I1Uz_A3E~>`_M9Gl(-Owy zwi?o=+TB zP`0AvW5Az6!c0szP#56c)7ja~=6&2@TlXO_LDXAhWn~5Q!5K&K7d(lPiHSE5pjsU! zW}=iOZ~ZzCevlaErizP-wi>ai-d4;Pz6A9P)`ha?v+GXPdNIQG0!cY|f|&&cto$!a z0J4FW=&JYV`iDso2Heko@W?MtFBh#keC`}RZ8WE@?na+d{OI?Wb0cV3R2WBBRB3c0 z#EZ3`gSp}0e88f*Uq+(gXG?z%8L`~Dg@^70Pkw7+#xU6{P_GaTtKLq>JM%85>*Z)- zxb~bJzlo&)yQ6>&;19IalNvHg%Ge&V(Kns2p7RA>g*Xxr5ZEnr-hECMGbZGD!qsV7 zyx}IfO+AxDgRI`D3f%B+`*?rLMabS1nVlV|F%T6ES{}%um32yq%c;w9^74{)c8|z$ z13ac_`N(mP7sZJ0fkJ_x7x~q^>Eod60Mg{G^3Q2MgWY|VL=awede=YLXrq(Yke{8< zYN}(6Pbs2zUkdBQtD;-F*%=*+p&hB2l{2!UKfe2`=P{M{nccGv(0S-!6+=a%ph!{( z6p6BiMwa`cGtYx3fsfS%I7Uv^%%&X#9)a8m+GJ{Grc^ijv#sWLPpqw3$#Jo(s;X$m z(~bt9*nlS(T#$IOtoHV?YiECN+j`KjbhP|y|>%~Vj>LE^QjD-zK;zQf?!;$|8As@Ur{ zeqz^q&|PU*SZ=nowB%^>UJonc{5D~|Na4$VtF`3Yw`-=wT8`CmW-4Wy*w)95o!H;T z4&@rRYmAo<^WSK8_i)ERlo0s1LDa9JZ%-r7a+l-n=2};j^uj!|c8D zalw`ImKW!a>>xEWyozh#Z(Me0zfbwt)s-JADk3tH273_dW3XJ7`DLm0{A-|iB$Sjx z6!gERZ5yo1XOrDG8b98bgQgb@800w>z8&SE^F?=8mz1ZcFd(|Q9yEFjhUB+Gy%!gU z^~jaIs|eP(8_rXnlUM3#+dT^xgGpvjZ$oU`rqD*mmCy3RZaDbj z-*z6?0I!T0EE}=>1g#-BBxL^eXdRbPJj{37faJG8N_=W!Vq)mLhEO>u>GSM00Q%0P}5HD6*hHGc2y;?J4o?_n5xrb{Nqt^M*3lkX~ct{yr% zt8T@x_l~IxO}01)B<--5MXIT&_yIvk>B)k6MoZDdUmm_ChjQ<`OpTaV$gFW{^2)g8 z+Q`2-NE)|FQ6qUXQm%Tv!?|JWTqv+LjCGoFCVqZe>71dg8ff-3dNz&8-(-9IdabM5 zE~5(#UI`nRhHpU5lq`<@C2b(ub7?;#&U=^WbrQ5EYW`0MiU^L5ME%bUFGlv#)S~jq zdrn(!|IH^l-xnoKV9=qV4aTRrQKFPAbWO|+ank~#14ce1rF2cSn6;x1Sa?KOW8TaO zKkQNs!?JG*RcZU_8&`{A_(-0Ly)V9>c>b1`kzS_4-TsqB_Y5oH(FUKyv{)CRTG%+2 zo)ghMm3<4yzn$XQzj!np9E0@I(&wdKU|4AD=*;i$Q5r4S#Df6m%CG=?-=Np|3m$j3 z$2>Jty%7$6b-Uey5~WWdxDXfqp1z&uVt-S~WH9VxyDvg5XS?rKgjbf@%9ohKY(iVA znl)khsm6{Ae@n3(&=TENS`+EtZ>f&jA)}9ROIXsb0*(=bj;XD!J@=>?%|-a9zR;K9 z+3f~Po|jGUZ{{4Q(Pg*JM*6S8#Q@4^Uo9;uLE#7r3W^;!-&p*nrr-zY*aXG{d+(Tg zy?}h9App^Q8HJ?6yATFU*P162%G*HyaDfKB){jVxw}A@^}$ImV=X1TBb=_ zo;fUMQ2ri!dV2PDEaCSw^j_qBMjmovvfcAN0A(E{Ka>z~@7}$)Mp5AInbZ2S2;8Tv zS)&kQ;{1)xru1R-3kCN*Ldk2g@1<(b$6}vMNwu~bO4Idm4(??BBD%w4!S|4|DIuNUwUT^lQ-6$pe)CqVUsv#5vG=AWRcFLINgDPTnNb-|1IH2CKc-zyTVE;6!XXR+`aJ0Q4^>oD zY8Fr`5Ca24LzUyIAx~o8*jNatC6#n@MJ63z#{kP-pG&7R8?2Fuzn{Til@~17m$#8&V>l$vKvrmoKC7F@ zp2W-UOO)pFhvpmD+$;&gvA|z{UwS4wIt(ARfPChg{IK2MCLk!Zo&J#;i@%N&?TJTd z%y~8C^K0x;i=n*61a1qD?jCTbpFX{w>@stWKU+~&w&6{WFzfjaoDRF%sG@fHN*1_{z9W{gRL*#^XpR;ZOysySy&YF*SCDsQ1JwVd;Anw~ z0&-Y-e*Vq+lR+d5VN0ek_3&NVchBDkRxc1j?WIo*Myb|wB=3$O;n?d%FQ zvZS$wr(c5lO@&tx(MWwsZnh1f0A!>LXyt{@oBBXfvWuqNI&QaA?=JSpR9&MX?q;4a z?iTIqG-N-rx{P7cK(}r>8q)Y;c^mWf+C~G0gP!|(DQEBP1jcQbjFRkM&&(RK!rxgh8sFp>Y7}U{OcPBX=tQ1`aitu zmz2=IA*o=Qe-lJ$pOLL8yM;@AEOXqWpxGlbXiOB7g_hq`0#tqLHY_P=$MUIEvHPd+CD^fi?gz}MXi9*?nB=|6^|CS8rG58The0-f(BDNvn-H_dG16%%lkzXa zLij**#s@F$QM^3O*k+u8R}i=nw?VCUSu1Saf-J@C*F@_dgBk`DMV`0tx8my0Fc*7L zTB|qfLKG5tre#D=@bz7$F?HW`aYtH{D$9Rj!FwE}UU)=~Ve(b`^pk_h_bzHdfp=WJ z7gLw#1_&__DH(Xldcn^C+Tstw87d<&@nIv;jHePpIVcJ2!v`FQd(0HJQa06Jp4QLq z4;zF#48YxgyqX;2TY@aY7tt0zdCS#130f^8H6!C*bmIOh+&xefkW9Fm3hGYGYj)b1 zz?*DFdY@aIEy;)mJ4AuikBQY=NN4Qp1nwC$J7rkH(b3V*ta!4|MGpSF#3>4*{Yy4= zgq7^NaCdF8HW5;9eKl`1f$sP##hHJ6EtxH39cW+eeU=9diqwfC*FL2cbPCAVjWbZlBqH|wdEO>g0M z3Er|>c5NjltYsP#$aY)apSPypH=e|Y=(yo>S15=)mlJeAy@8Z#G+(pPr{CDY}vx#>Bj&53 zTqz{C82!QVL`n)?Vi!GfYH$z}k)4-!1)-*)K_@Co z<>L^BKY&?2mM@@j^XAP)GcF@rTlTle4-92TIW49K0w>U(iNdw1$jEjRhR4QkAeJ^a zTZ?-{auZ&0k#6_Pu}y2#pWGp0)?}YBeMWNc!2{IpPC`nW(0>B%x0$u|d(bi5$RMJy zI*i0--$og_5~Xnqb)Jo$!Mv? zyw&nLBnt5}?~PLH2)))^ZMcQl`mL%oeY}{`it1Qzw1UYagLdu< z3r{Sf<^glCZ*Z6=Au(~jLES?(WU3mXe}$S65I3?tH8C;K_bvVv9CV{j>>(2q1S=4- z@jUof7+!OF5QSi(f_vttl7&Bw3aOcx7AIESoop|Yw%L} z<;!$)E%8lqhn3%qohhOQ-NwAUyt&<&?S{!d07nRkiPQg$v&8rFNJ~rq6Q&`@1^H7k zR~VoWNGbfl4gB=8AhYnQ`*Y-GV1$UbmZfP$x{@qaW5K;-!q=tR+v9I~zQ7YuX7hI9 z)Y*H-YMQUageTXNV+N!ma-DgNIGRr+BqV_7%NQ7>^!)t;i`dU1GfB`nt*T1zd8Gi*(c`;mi%!cw})!GCrKvw{9Asl+a+3mqlORP@Ojr*}`Lr4T0Jp`#-(jA^NbsU)Y1C6NNzv5+YJ(^7 zCT;)*s*c2=On;Es)ZnXqO=8f;tP<0ReUDICcXW5J9==2(cdxPAb%SqzHjOu6#)7YG z{MO@lB&6=f!6rHaViu0F`vEQyn_F0*A#Cc7=~Yx!K}HDyn|(0*9wCLK@K+?8RYXpy z{rppKCS{x_>pV|AV*S>}D^>U1!FPXHO%Yhsq-JklG_F67T7Hjsc;26#c|12&82>?3 zIu17pm+Dkafvmr%(zx~AYK$i%D@&=;DTzTHq8joST4V8=W8UK9u0~TettfDK>Wfxr~!08$RWLl92P>w3R(8X;>nY1C@pY62pCVl z{>4&?DI%c>38bh9V^i=QCRA!OIyQEv%p)3{K`%iC|VE!L{cj&$)ERr;dw>`Q~RfUl#3L z4hw?*@^f->K{bvm(?7|x2SSuR8LRoLD=WSI{phgu#>NssgMUuF?jQhC3@SeLV|M#5 z_!!W1!C(csMwxQ4vZhA&j%CQ`35wPi_?OA)l_V#vmST}WbA_#(Xia@Im zp`ndNg-f{jvV=Xjea51sOPeLy=(Z~{fZu>eG136M5WxjKX<*mV#v^GwDA%E>Myl($0ix_AUd zM`K<*2<9Cq=;Mi(*B7@3t8XY@Cjj!;gi0dE_sXm%a;m^?W%pSvsAl~;JI46^;J|5u z`3-v#jc<^EoQc&N_A|^M&}v;EfAgYzY+hgy%(|5HH)a^leeufSWtD09U&53Z%Ke0HznQcC}DB+^8TmG^krR-9v+=?>lTQH z0WbYp1DYt#;3lv!lxz#Cw8+KLOT?Cd%qwH#R9Nt!=h`D!fl!c|7LQn*K+3*vaIi&L znxWub9_JjC0LlUfU}y-LTa+VCN=7!{9|`8n^xpCS!Utj{RPav}b+bB8#Tq+$dJf0< z*)=bUez#Ne{K2b+-T@ebU*}OgURz!s^ThnQcKEwjDjX|> zugnE9y!e|poG6W7KlTews+4EhLB}x3Bxr%ou0tMAg<1|F)^~vvkYOV_sQk5 z>go_nPFD#H+|Ww0qx387Glf)wmW=Nz&XfYiS=CpA&6NtusA^qZL`&tAuttW`Rnpm; z-1X?qx-maY`!CvJjz0eEoZ0_+P1nY#mFEWKJ8rv>37V9n2-J@Y|j9$>ILf_3-bVT@O#Ri-0p#C$8TjE4s#kQa?4XRMw&IEoUQ<0fXj{<9ZChNJGI< zMdkL#j~}0@t7FA=VCD`!bRI|HLE&UQa;4nCm3=OOOgxbzv5RU5rO~Gqe<{L9gXrUA zt^52`l46#1YCcXc6eh}b5G8vFjxY!teiPOjn4m{efF7c6LYKyZP!o5AgcjH&^Xs=; zazSa>oT{e+W37>;d>kF|2@aGXAiwmu^fLqg^2Wwb2$-RQhQSj(ZnU;Ucv{<;d3iXn z3q%F{>UYmcOFrIRJpG)tRC2j5Pj|~i{ z`UeIIVF4h!HD{hAx@B_kBq|JElNBmT3S<~XS*uD-r6ZPR=H@D)FZA^F3!a8u!wryl z?w{|O=MI_{z|K@CBkJ+eE9YZ%BETy7?Su|qcySePmCMYmaf+H5#326sS-N9pA+vca-R;)#OD z5nlNrHK5eL{*zLhe&;4MpTV`J29AgKM($S%N+8IdKN5KtaBzezz$Wd#?1lA?r8k+e zSQuj2;9+9COr@A2F>A%b8ja*8!&r|H^-DsLj=;x!VQ^+hLGSxrWyph4JWA=i_{m%6 zi#jvX-$jk0&niL3VyD$p=d9fj_yiu*rHx{i5UnCdks^Glq)&JyHr4AqNZ;KcvxU9= z{YNNX-H*TP>jX{L>!3Jg8Y~{UjKs%Ws~*SmaW2#Eh`)QCpUCpQgHuf_%^s;qQBnQ? zQG|fo5e+d5^0UAtXp-wkZ1>*-^=rpea??ZS)uNbqZ*5XN-TbUeq~WLgL}=Le4~v{?~T(vw8)QN#cC{-d<$ENlUVhjG7o@I{h7Oo|7 zWSHy?2_7oiuIuq)ADNlWyfQEd4aY989a$I`JlQOFu@NEtxIp2;zg5;O&{*dTd#et+ zx3}QN^ZLynG7NIQZ$8gzMr zynGBBA%vXqL6v9^xlf?=)39i+k?$}}Aj0StH$UwznS8AG$Y{`wti;WTIw}{f%K~5U zmoASlM(?Aaq}XU6s~IH{>a4q)Imn1HQaq^7&lrlGxNO=> zI5<2^$HvCCw0r@80_Uc}dBK3^;RJU1SSYxqsly;!Rdqo8QK4EG`o46!@`bsLjYikz z%F0e08S{# zkYzmVc!Y5D$-VDKA=a+VWHYrXz=Wv~kmyq#Qae1cH#1d#LAez2mtnInX#HZ3eXEVC zrUSh$>0a0Arhwt8*X!fzkj&1n9M6V#2bvmKM7QcYEnj#(6VSiZ<9(!XoP52#9@~JJ zhxQi{@e9ZmjBT9e?@?mTHu(m5vgt5c4NOg?e4o+(d-&i1>$ubSjWdhNlgHNdPvaR< zr!Ef4K|k$+y^R$j()ArM*%Qm4fq)u|Q&CrM1z|R06B9JV!TK0>NDD1?6=)94&|3&? z#}I#Q`QW$a;B}#<-$T7+VDMw>u^4s+>l5E!_>vs|Ih&lsNrRyd8<2(LA=OHK4DU|B3^kRr_GP7)#S>_Ysgv{QQ zztYnFa*?e%52rJx)ny>0e=3xLrtSL`V%r|MSXRV^gYrqDTj3SH)7Z;uq8XcKV~Tyr z;NT1wt>!sIN=u4!#>6W}=X@>iXq&Lyg}Luy^Nk z>$oAA0rLs^N`KPS!~_dl);^rc%l1Mdks77tAU+~`hljJ=ZfZjfck`{5POLjM&D&2m z;fY2M7mlsNVNG!_uR0HMNns~_{p5$gC`n2E+Kl_X=@?8c$I4jx6#_|2OV0VXiU>!;IO&7Aw11`O7e6bK5|!enx+a`;u1ka8N%o6ZdzY{lonkLVHj_ zOfwS2@Cg~JDY6OTTZ|e(@^J897Ir^qfCDWG#3PoO?d?yWI|<%0x%Qz*S5m7wKkf-O z0y^&yI=C>l&%a+^AwQAN9{F9FC25B$Tuhz8a5sUM;2YxDFU2EpT8d09^ z7W5}CikwkNwRB3w4G0(7xe9$|_!mVQ)=8uc<+k=KI0KzsP!Q{FV`CH0V$^pt*c1Og zJIp$t?O(*o>gxPKFf$)w!m^}-&ahB(E@2fqFb*y*O1IVbc5@Q|P2>?DAK$XFE`u9L zL!78t!y0vy%;FZSQb5W$h}7^c3BEZ}57|_vltd88CzY7V&E91+?2qNHsd1(KQ}z1E z-}T74H=FX&6K4KH5K%f5$dVKo+Wk04EM#;Z;y*cJy?ywl^I1qe(>l);E8EYi9EHgh zTeNAPGI7)}JdbmqD?WYd*Jav-(i{@%UJuX%k#Y}RdITj166o`QA7^K0jN!E`@3&Af zn=z&XemIl}3+$$JdS7l$CU~t(8U@i~c7GetKz#U8S$P*0HLUq4&NZWI>+-Qz@iDBIj%)YVfD|A!^zhC3QNLYuU6(Z$JCLJOt?z~jPKE1;b zwQ9?!Hv1}s)U39TAGLM(nXg;ijyD!t!TI)?^?=4aSL&x;Vi3{Tt^F~GYn57?nB0f? zn>UHMdmK%8-os^;B&f`67+MzV#B3aCN2D}5K*so(yGZgqp&DaWms0#vC&T+Kf)g($ z%(({#?!$8%6&l($`bAX=Mm4V@`IF?|t2!*rV#w_+56Xw!_=4>u??9<*hjeJ2Sscs6 z9va`lczy5IBPAjBg))MKQcyeauf-mr0o|9tWngkmb)G6JkCE?B;q;4`J%?xt zgBN=`g?Z?!n&Lk*hoK?0tCkQO`s>+Jt8$B8nxu^}68!|{#&?ND$=uIG*^q>9eyr@B zSFW(*^poZ0ew^xz=jZaxI9AQ%&d3dBu4LtKPu%PjL{1H7zl*e65BwMUECyR{>%x%8 zvhuizE61y}=q1~M<|M-TsoU(2(1Uk2<(&g@yRv+ zLQ5XJ&DIZ2-_JtP@dFR8`9+NcAnjs3tyFEQCJAn|iLU>mtUDreH079$Ryq{!-w08{ zx_@0Tav#&j9~-Y->jgd+X{a<3I}gZh?$2*Svkn5DPJX{TXhR}JK)sDjNePF%AxfeZ zC>GYD-7Q$t{8j?Or6w~GY~9Gt3K9by9LB3+tSsC6_kK-JPv5FXPhc!|brD6OZQ8)5 z(-wLYc0oWdgO#?h$AeTI6BCp3!3MN}9>ws{Zf`dOzoKqI2#rB|vG;j+K;x%tb<=?X zG)u65&|n+XH+eu?%vvq&jv|zYKoA9YpP`snHSnN)6Inbg? zw`}46pXfPW2(*qXrJuL+kw3<7@0hG@{Fsv!lU;YPkpL>Y$irRKdkuc#G1bI}Zo#!16#`*~ANNJtOGj&J}qwWvi z3?IL*^HY~E4=RX*8gTIYkrlapVGimsv;mvKkPzOB?}B;ITfT}fMFnB@Iw^B@5_-K& z`2GO7x{G}zeCfnCRqfYehqsX24sJSCuIUvq7a0wb=?QRKYJ%22_~_Otz^&`cUJN|{ z93D&++=P{{Kn*r^+daHK;_iUCXO^RtZ(hN%?nnQa;bz*PZx!0lx6AU;Uj#Sk({3iG=5e7eXq&~$m{VfH#Y&o{TnQN$ncUOZCr_Vn~)2%rPq82}?etI8+ zXqXPn%Z@|g2NeJW=I3xd6X!v0n%nb8R`u$EC+ksphJLyUv}NJn8}Kot>bZlf zHWpyuwuVz3>PRHV#Rb4+Rp=I^CU?VksBDy80*X5F=qZ5>rx2_l*lJtO2gF|OZh7I9 z%tnl@gY~MW0JplE@Ks2QlFt?ayD);=it~RA0I+9+DZ71|@B2cigjJcs#?bHxMNa4RRZEGq5*Io zkuBAXOUR)Lc{mt}*|({z%XKzv8U5_n?Vgn>#nTK6^89G@#|FV8XklR?_0sF>PV#bT z)C?8II$uU-FZ#2I0s<2}TFyKnEmISdws-@4<`B!Psdew-l9O?v4-GG%sYK*=$}%9= zAU{6G^IA@#IUs`X1uooLj1>dM@=Y?HU%OJLms)u^vaqio_~Wm+KX_kC;EZ+dNKy-v zId>xwDyYMml!fTOb@moeRc>9l@TOG|M3j&Q6_gT?Zb@kjQaU9C+;oZx2#AQ3fP_*~ z(j7`mOGzUk&89o0;FD9X2mX#iYlAx?LsB6WwX}?nZJv0C9n?85UOE~%o1*cXo zXI)%Pj~QN8;-aQZ+>duitp9xRWtgsaU1AYt!^?cP?T{txkRNg zf1fBED4m3h4LHBid;jxSL|bU=y}MU8#H;BUPU@Zls+5L?=O5DT>O;=`wxQBXbh?|U2>_a5gx#2Qq-5BPcu^96Z<^u@4w5=C`@b}e8{6A zks-<$l2f|b2>7Jz(OzjCGL}LS-G}bCHJzP{#^?-3cOKVe&8Gy?{W_s~r#e>4Gn7(3 zu6S2(*p{D0!siK^bbM%t;(4=%kaFVw*Sx&lYk#B?l0LKt`>g}c^4pDkQ7mO0a;_RTn4bbICgW$%)cN$E|D~a!**+xd z6-kGs`OxQrDSQ{M3j1Mz;?%xpw}KtS_m=+)e6n%c*m3S>RM1LHP&x)Us)?JGz0Xe1 z;{|+A9`yP*TfM*yA zy6PX~p=a2zdEf-6mwemRM?oFsrM5Q)(XOsXx$YVJpNTXrso_VgE3v)1i4_N%?OYp> z@s~}O0rYAEwbzHh<#&w!#7!sB)vc&znj>U#jzQ2a%I{^kIbPJojw4GB&Ju$2?Rc}jQcnDe2bVK~57 zg%<3@4;MbOvayAP*sZ}}Bc;87iN81`qJGHSahAHTPf1m8G*e4CI72#1yUbqyi7kSi z6?9qA?iq!*_6?#xT@U1~?JTnFMkG4mi$H5Eu&Hp91((K5h%SRm&JYf@Dc>% z%76a_I)Wpxwvx+)EW2o6E-ahN?#o;uyh&>nI0ApivLi*>(I%`}3O~J@HDv`0)v*Y< zyb072r5v_k9#rd5{FMf@9EWx*o zCIL}(a*Y`+M6oUYAE}r$c+`_$J^b(ej6J1q%H(XGW{eiObbp%N6sVimFvn(b#ql z`OYBRLzlY+>k~~~Uuv1Swr?Ua>9wU2KD5Z0b%}MEot>R12>o|%Zs#`|Uy&#H{MS!$ z$rnDDz0X*-&F3V6b7xR%rR4pa+)@L)WRLvcD_eI9k2cIm9svm6VB8-ld}Wr7x7lLI zaLf|{3jO>@O1+|LmReysf!p+T+8i$14P5_1RW$>p?Div*MBv?QFYr3dRS^6XW@cil zK8+*!fX2tebKj0DbIRFj5)KZ+{;ndxF4JYLUs{ILNwwUL8GjLe5cV+4;Z<)I-Zx1+ z<*{!b7d}bpXoz2aR5Y=AKx_iaj6Pv4AQ-uHoA!j*#1qtu;6rcp9|_D^od24j*MB82 zBS`)$fw_tGUpp56uLNc*!GCo$4^aOp(*FOsh(Eo{4)G;YGy&8mWwkKaCw%}5oZA+! zRXc4O@SL;r!r-V``iOhILfq2-$#WT)6%Ajar43tH0J$E}f6&p*K0vtCSPBD#5O^a*56b%Xwm$$OYTr>q} z6*u^aaXXM$=*?w;ybdIVSUEWdIVj*M0dfLhVl33==B7GuoKBAROn-j3|u&1^*#*0hd!a~iw>{3Z>VtSSPI*RNhJ`jC*2%%c>T=uHtbo0SGoAS(w4Ub1-( zNN2(!a(0s&2cPI!xm{1Pl=!g3COvG^J_BnJ!fz2F)(8@FfN?=^0oWfHLOH-#OS=p& z0P%#lOuzh@)LxZD3I58O42t~}@7{@riAn0B&xkDosEij+2Sf`9XmaL1zkc)Pk^!OX z8TOx&%^Ux0Sv(qKJwUeU^OrBUfW1lXo0AKr#6p1@r(=0O$jHHrRWj5R6lj!%W}e-| z+z009!-tVH3P8XL>FMbqzNoxP)ZG30DjeQm-dz{WOh0`5_|;P^;UpwH+yS%ZN{ux- z5gwFT^sND-xPTRQ}3k!Kbk|~x`OHYr@m!Fe@j_xgl%|f<%Cx+&Lw@XUO*UK38 zaldEBJ(MW6KNWyb_F$N4bx*QDOvg@7+(d0b!=rl4>JH4?Hw-zJ*&0EGyCOsvHW z1=aaW--67;Aj(htbTiqShV5C59PDh@bn1EN&TKC;Qqj= zjGKa79inC6s19gYPPJGiMa5Zogs8j1!q*`FNjxLREU()al-gRh}a!gHV z)b|vOYW13*ofSM`BGP_TT91lE`I#ygW}Qb`vZYk~y5fkJZO-1sjgW zw(>pY@dk=KjOYt+@_{&!t*EaA;b3Kb0ryQzm~JY^GLWRh1_B9r*U=@`cX0FvU^I_3 zDM|(;tt-f>xJ`3Q5OuT+!w%p#Wdg|`NY5P>LPw(%bIjKI`3tI3*3>d$pr*_LXiyDp z?aU*(3b&2ahV$QsjTG5%*vmj1Ai%Pyu6OsC7S~W?7{;#4(zMxpeWo=+fPm7pSF+nF;m;84uAxT*dow5_h8Ksgw>tZ>SG*(KXGNKR%4s z&Ogu*Frc{~)ZLNVs9bbIvqer=&&& z9yhOA(r%pDo-F18NRZB%?}LvWpT)Xeq7q(Sef{P;^&vKFq%Q29)w06l0ExzG+%KQ( z-preTvyX`Q)05sZ-e2c`o1dQ??6z`V+rYrS;^N2FH{D4RlXEhRnvWknG8!vKOeYHC zPdRp(f`Vo8h?H55ft_91XRY53SlXr8&&`3Wv$>V{>C;=d6%bBLCu2JPgKFV=C zvb9=yKY0+>1b!11DuU!}#%s2V%Qfx%$mcZny2>t7tF-^TZ9)@KYIVm2hZ_Xgk2CtM zh6V=E%@*uO03=+x!;!iFuEM(S;p4|?*=9xuLpdKWBgctLsE87f{F4UFw$NUMSV)!%h@@_^JQ-la0p%Kag4 z4*SoO-P~Sap!RE6zNMO(+m&hwaVI@}IHZUVw0WE3j%41&BqW4Fc+UWeQfndE+yY$| zSa;v*Xhibf?zKzLAU)6J3VRPn6grEgK+d{{Ef%j;v-C!@`MITzZ@^K-=tsj#i~68qPPY>l*o>2O$ZFMc(NYt+MHRsG-5g)dc>KK|o+=b$6^}v8}6X zkR!izvEcHP;P#4U2618Gk;UY?wcDU`1}>wuwe_7jvjqW;R(?BxH)bi`Z}8m}xf4uA zY1&+GJ1A1Lx3!#K7oBuVW0@r=`R9`5cUuu00%F`LQC_K2F1Q9*DBVt%CUAX>VJ3-n zgxIiB4&%sa-BEUarA1sLFgxS-ebg9u!Y!~q#3svJ?2vt6d3Jh&+LnqCeDUqw!~cl| zIMo*NL7@4?#r?&_8o15l+y(bO@e+0=;vHj4h|tatNLGPj9^=CmWhEs-alrBeZ>Vnl zFQ@i{Cr?6wfe|-wSA=(Yg-H#6U+H7NSy7VSPrun-? z5hEgR`F1JB7#z{c(!E)wH6H8LHhZe9mJcw_`nyU<47Ik!`NH0`-k$ICDDBPfcRqQE z*qpBfw8VIuJ;?dMo~qK`94h4kmtd#9!~BAg+e*`$Yry-ml$eIR+hwn*~#WESKaYt)H67Jk8dIwQ$#=_E-o0-<8C$r>}2#G00SjNsc<6sO-0S< za!nv?i;2-;Zug$(ucP_3bB=&A>R7&n^;_*23ID{MKlN)}7BVI8s3M#gH*csE;Y)t3 z#@SzKSu_Lz>wQ&oE;=s!oOu0U8g3KZ3GY39eI0+@+g8;dZ%rtf%8d@!3tG0(6=6t< z*w8Yw32KV+(d;@*zT$w)hX?!i?9aW9c!BOeE|z>x6J@t!$z@NhHlnIgDFpdO==1*tZg3tLh z^>6!;hXRXk`BXFU;@zwMxkdF9e#gYOI5}N(o$ZL;HDS8&(+cF`TV1^-Ce+M|^~3Bj zw$0PW;Cyv9fAEX>a{-eK!X|qHcPhkOAw(w5El$RUnCcmDjw zu7#M%oKX|M*ju-EH<=2ZT}*8G>-@|0i*>F<+6ZO;sRasz*a(HAmnJ>S;Of#oXP#dn zjHy61omKcXz6*ZQotz+?3i-7Hv$o4#@HPk)7{OjZZ;OV}_*KW>>48D566H%5QVonB z;W+QO<+#vdO%-$8O2+FivgrIpdnOz69<~7Fs5v?yIlN;1g;oA8ak^O;f5hH?PPmJv zCON1Uzvzq?z(FCrY9yClT}>0oqShSCZR%C>bQ1-kDrrd)r7~CLYs$2Y3}#TMfie+s zi(ZLXs&HhsFlt1Bww2wYcD&4i9%9wmmvX6k5!~whZ?{OpEq%S2CO7UUpFoB<1+EKv zEtTTKV4L;uQjkBs3M8*aig#EEnK~ivi#4npcU0F+JD$zY%`x7+dj&S92z^Oht%qeH_sjdKtio1};NGxA ztc3?#eq*aY|9zS2nQK6P7>+nqESdY?bdXdSdsfw>BG8ru(HM|+jiF`$1&|*gOF3rm zHd6c)SePv!=*-c5GwX0VgiF+i=pYbek=j>zynIh$qV~Gub7*{zu_TI48wM;tU?P1N zJ7&&L9@X~^PYPHFAJWcliNmQ7K4KBw_|n(@E9L?9DFUOURuH5QY8^Ue6j2mY%P=pN zzIX2%SO!#VW}wb+PhX!FI5Q)U`R^gWpT$&1_Y#wkv_SUDangIQ;)(r4wL3VFCLqV5 zY40QW{-8>-{EM$~i%We1iD`d?Qt{q#m=Fy-0@!bGiOrDkbLOWc#MbAY&=67m7NmS1 z6y$NC?i$g-YT0CJaq*Q5)kLRq2T%kRbecUU_m(CYh))jo(a2?6G5cnB%<6njXzU5k zpZ^XTTI!D;z0B6i|2YcPX40!y=Qj8SGgCZgj+Ed@{ktsnKZ7SgIAYMXKn}FH zK#>dFtPE|NFgFo++`(p@dS>H zFw43pmI@;Rk4TZ(hH~?;jX2hKw};ni`=Z#SQ6d0B3pH z2p|(2bo@e#X9U1skj171VQ}PWfC`fW8aZVhz)wW}9}>}~pa27)55zKIY<}m7kM!4Y zFVEJ+`ZJ|UZg;g|Dm^G#aq2=B#|NsWt{R*#9s+73xOy;(*oivyvL?voros^ylKhcU zt}sZTo86yhw@GsI89GLS4lxKVVH&HUp&>*-4SAMQhsm{PXQ0;o64842uRkEFlnb?UUJiGu>RHaF|J#)q0y;R;Fd@Ic8RH<>r=XzB(G6Ws3Fcn*04%qE(W)kGq02alt4&UYl{@b-?pn7P0miq%mfXZ7#6dbyc z?!W$|!U+!~9k%A|FCkw?3+G!FKYZ}u5+tq=9!*0aNef8dtUSN>5p0!-mGvA5PK z_sh5P@kc;eB|@SbrWb|0B#zg?QxAU*7A`twK1 z>PXV$+p!zIZAJe8c zS2t@RVxOLiF)j+*YKF7b?W^OMY`U?)c9o%slV+Ki za5De2t6kV7KWQ&I>6=JM-}5y7y^kgM(4Dm9bd2VXzl$eTfDy{AZvElGNOO1Chce&w zuhHK;><`7=FEhPxbL~+5^74({Jj%+y=-i9XCel3;IWg_s5$AHW#L7wf=;`rMo7bgq zCz!tN&Tif7?{hQ1^QGv%&40+mV48qy3xlpRg8JZ*S7g*zLa%+x>d%>O!3bcZ(j~-+ zU_Ry!Q%$6wt#o&)4vCrnQ4h>vo1*LqCMx+@LLOc*_MEn%nWX;1D`}M^rKVD6o^!Z8 zDIFe`yo7#}Ph6C1Rhg>juTnD^Qd?53!ejP$oMN;qL`dPXXcmq*>gM*OrmN@BCqFjC zI$3{y?dKq|Vi7VqfN?`>G2qV3D|q=B2yx>e1GZ7b+;h0Ix9NkXofeY#;9V9K6P78z zIsB3jD8{njI}mEv4??bFh-*p2wlA?{FxK5D7P;jP$u0_LyX?fjL`_@iu=W+uZHM&VAjIPiIwY6_vMXxd~f4 zRo{iO&@OQ+aFk)uOj`1xwDReK?64$%ykOj4ZPnjg3K(#r*x}8zpJWFC*1uktt&&9D zF9XXLRctVfTHhUYFohs4R@9vbE6Vg8II^i6`xv=|zJ5wP0mF?47&N>A!joFxy{b5o%hj8rf#64h|mXAFj z|Kog1JkWwl*0fpItW36kg>!)erolWV?8YUNs^*q8-z|1d0e$)7F@^1_9eMZ#?VN8; z)7TvMrZWwz6n`>abe?LmKB@0$>dQvA!t3^~Xs3MqVZVd(l{z&?r31Q)e_`d$j4GXB zp#<0S4NbOjQjjM-Y48xbSU5l?Skkeu5}{NXi5t;N$M55JKqx6QL4wG&1p|c4v?G=~ zC?uo-z;+E*n zY3FlREWx*TW|faQU$AAPdza9EPN%{W4wJ+yR557o)fFb@+q%C=n$UI5BTFh zgWnA7^BE4vOkeA$E6JlcB<+s=dXC%0#OEy=$j_XA?-tfyc!@6{chD;&d0gk9YyPqB zO??{X!@!*%FpR7}f7QmzjPoVelvxY?gk{tVD}=bEHO17P&pRKy+u#EZAq}0xm+v<0eZ!sH!%AIOzsNwTMCjA_<5f>~eI9@X1(I8)38IMlU@At~cwHlA=tr zTRm;Z-NUMic2lz++vum*D>0p6{x%2nHdt1M!l`SI%_uLd#y{A zoh!U`Czy+U#@wAxFB5;*dYsLO#46R~h1f_Q_h(LQ%KU$Slc)&T3YhDz=N6Bq zuEAoLu_zJ97kvvp$*kT?^XDD2M?q^_S78U6ak^+KIk$4*Lb5*AgR1bLWDVRwrV#6n zg$rQv#-mG-mCAV%cP#Qp`zF+?EtZniNiT&AEj#t0-PzlGOjE1Jb8Lc#2l$N#0*+`D zANJRK34&Yvw3JMpZ7?0B>6uQMtjI}@2kz%Qv9II|5U{*KAyqaW3g#s@_ZA=i zY^{)_klW7%-6Wg9a4p~`41uVCcM?&lY!B;CG9H1@7^&<``BU(zKEARyuNTMU$&Ehb z=-jO{+`&9LK3*XRF6=ab?I|dUI!h!vQ(x&3laWBF<}v)B(nyac|Gm%Ei<0e5AyZ;L z_SY|y6ZO`xSTq*SQZJG@v?O#~bu;qO)WTdi2+~rLKitEFLkxwSZ9tIrcex{Tx@zKA zHd-t{V3E@RJt-)!vC%NgKa=5wt%KlE8)?}xFMik3YRKlBJ2(Frw=X}S6~Q z<*|cDa=K(TDljSX!~iX47ho{}9@=}e9KhVBtcG`-VcB$g_! zBe*>)W}RC)l{3q9n;%!BfU)qCFGM$i-Gx!a3`hWFZIL$Gt^F%MmHT-|?z!`;X>9(N z(CzsNv5W+>?mSc>bv*6(@g;j?{*^`@Ryg%)%@-wQBX>Q7MDB~%W>6W@IXHS=K2hHt z-(33Wv7^EfZgV7OGe7~8$ssmmJVVgxfk_@M4lUL7os?i-pX|?-~)jTt;b@BH> zO9@YpW`HePe@XaJ{+u~B3V)2W*A#>@kEVTDkL;@r_p{atb97!?F2w2~1H!}LGPtbW zdHGrZGiiSimr8^&524#%&T!*p_Fqw#GUF8L#mfgrBkMBF$t=uir+rE+?;0gtZyEmm zn(;Hu-a|VvigW#v!`9iNyi50k7siZfVlxfGD0yx-8dF` z5$~uzH~*M|R@}x!%%aHea)eMm?g&B?1r(chgg~41N!qkB&k&NWVP3OhDbat9I zu%#_xo6O9Tnh2F-F`kcm%Av*Y`Hw> zMngW8J-Ii{nf~p;+uJlk>z|Y><8s>voSjQ(hTU)UKuU!lNlpQ7c^QtHi+u!snnfHR zVq#3U7kWWuj7vc$VTWTG>@-Woe3=C`Nv(I{RZw44bxWRgT8+7PbGSjEf1qn-PZKnHw_VP3;l=-tNqofam_ObaGvlZJL%TtI!HIG zm;L$5<*H^(K6On=P2^`z!eeuC!H^Td;$@D&?D0=4lkIAqF%rD3rCn_PyKiYlo~@4X z5mBdK^g4gx0yc_Wzw%Mp6UY{zrT~(*-GU@0kJ)1^vr;=`#2$*m z1w}XeLnxogzZ^9Z$vm^Gl&o6a6tEAN7m8#1R{r<*GY^)>s?!T5o`9{WLSTNZ|1=x9mj4z(^Hg_jgb$|9?x^y+0UdbruA-`v+ z1n*i^_E0D7%u!0-XgTeSq=F?5b7F4y{^7@g#rV}o4G8=)x&_oH8oC}B6l$EZV`~%@ zJY>3gY4YF`(d$R~PVwfM1Mk&I^)ipv=Q*g?;T4y2fBkAb=F+Xrs(0i84jf0B} z_TJshj?#}&PW-VeQDnSklD0YV+aEqA-nEEcveBJ&a*k;+YR$ZqZ++0pHI*lM`$j>e;lO?t(p z>UDi6x0#^py**Pc@|VMmP153VK)x@F-OxCB_6fH4GUk^Un4@FU!jXyY;WTQEp&f8E zKaVnKGbwSDIy{Mq(6upm7di6TqNw;|6i<7zP&*OUAaBUhOjz9QnZv}~$Qw*D zGN;nGEw#$JX&hYrP1vc4>xEjbi+Q$Hqb=Z&@KMn>e?K(6g08-%v&E%fgLAspwxjo~ zJ2t6(l3>#X@u%=cRe@5bhZeOP#9G?D)lRO(93I*SB7XWuU-^Y&hB(C4V5 z3{Q4JIm)e8tmHgS8*#ex4!T5LRkMt`OZ<#HQLMwj#x;_d%F`>vTcaf&9F56}^dB0M zX}muA&SfZG zxtBQp!1=IcQq_?AVsV%q+xSLtt~tYl0KMJ;1@DQa^Px+>Bqk|}hIFdSn%gXKH|P+j zhVny>QQyjzL~1j1N_PS1>=|xZaib=Mx!DNz_vcc(sntwgj?>J21=?zPlhaYdj6)wZ z+bY+&&yz4a{Z>%&E;by~eJZTlp(9UPLR&yd(rh#;hM>iKSRpnq_3?sq2`)*s&QOMq zdPW02Ls2}_%n1a6ZhQ}`UOqAG`?W@OeOyI^Sdh2}%~vBb1=m1Wu|S{;K9O}kgwn~#go z+h5YpPyCRqq7^;3L`GVo^~g4eQycAaFiETCnVU_u1_7Mx@~kVE!FAkkd<`wxu2@&G65E%JWjE<}HHfJ76@0emAzC zi#iI580_ZOWcsezY2+2U&pGCdB?fIu3}T~vcT4z zCAgFI5d5@QcH0e*3FJ?io((=Z2o|P5@RSnUKZB__mDoD41zbzIm|C4QNmfD~B|>!` z6mStuC)lWk?WUzKBRRS6WoOzGHxJiBh9(syPG&qm&RxBrk*^|?7yz&-4b)SJT%p~m zrKwoHKrqi$(jKaI`PLFRN0u!+LSX6$d9dkuIpML4K|{S1AO0taMg<|E4sIj6F_jd^ z^kHSzU?p6&ErBB)RrONPlwB{b@EI!{>}Y?|j_}?o z%GfNH)5|R&&oA;|GG*^2#oF2*-iFju&9Yii5#smhkS+p z$H+9``N8?)`pIqG!I4rH=dp-u*4*nG%5$>F5?tG)qzYnTr~H+N-o>>@6aTc`Tbf#% zE{oq`EfnGA+gK_aq7XqzeiMw(@`{Q?Z@iDn+@4jX{9Z6Urr|L zYb!VjsKcALlyZUFgCyF_p-*9F&m-9G_x5Tn4`OQ?t zPKm>z)t{`S4Oy9E9b5aaBGdY_u;=do$hp$vUx5aj4ri4vy`gt#NcEm9|RKZhPbR`4>jz&s$G8O5f88Le<;K^_|sF|??Dg*0CQ ziPqKAT-}ljP)u?zKY?NZsXLH;b|csK9H6mStT6g(njOp;BY+`L-=Nhn;1i2WN_2c`#V(E>ZBWX} z^v7YR)zI^fjMqayYcok&xCWH2q0T1KuMOI)sW5>6rmr*yC^zY~Yj2?QB2^Fp9v&kn zCvibR0g54D1*Q`4%qTz3WVAWllcyCqHkp{1Jlk1%te?z=Jqcd`Ogjn%uuGSf-~PxH zPYt^LU{JjcI(UIl=ZizbY?XQCrCw?7vx`e)K*~%N4WTZ6Hgvh=hQ0ReYZvGV4Z`;p%OFNE~JI3gq@ z>@9OJ9It~rp(KF~@PrevQwZQ}P&elx!j<}Ocg{F>ij(TcFpfM5z@BX`cbu6OrxW+0 z{0ADUIDUagI8u`f!lpWPouK~~6qIAKs34GT;VWVd^~gdmq+2XZPHTI6WSoH(SX)51 z_z@!CC=GJ&Fxv1B4-XGWSSITY>cq)G12IlAd-C-zup&em1WXFl13iE;j@|RZsR%1H zuQYA*@3I28A~6D-ReR+RdTz*oI6}#I1^N|x?yf`x;3R>JCn93n+STPEr0@*JPy3q@ z7O+8s5CRo6a;JgYf?QL&o_$}s3PM$;p`)jzU3H;{%H6GidDE7+#$YBWze#A}nciMC z677-Z2k(|C28{;=`B2^&k4`qwP3nh3-cxjYuIFQqKt@uEZeE z{cPdh^ilzs3(~m#n{k8Li8H8`LKTxOCO0Ri-@HQB{Y}*SFn~ia1HKZ*3Njw=%^5(- zweCr?-YB3xl!2x0HW#He%k4Vw^|5{aO%Ky}=$eg>^U{2x#J_r0Kb^pkfu!-z5;Tu8sYACL`&l1P!vUMs$~Fz3shzl zEphZBd*~&r`nMWGfnPm6egKMW9VQ$o^+>HrO78Nq;tOCK#I%_{ zGI^I$HGDOfbN*stMebVY>WN!u>RZQwdyf-sJKWg!Zj^Lpd6Zya>QPf$zoOn2N2RTC zsgE3Se;yA5W7Q@mmsJuq%$*D!SVdbb;HPPP6s-D8o%O6=x5+{pl2-i_aY_P1f zbQ*X;krLEp07^wR8-}}g`;En{U&vnFlv9Gzki{yb=p$u*aw^?_zosetv618qQ+BiQplMd zuEj#YYf*u$xWLQO-2Sl}1;}K9HsS0hqRPZ!=(;*Gx6QZM5h@-b@Q-MTFB^g)*8f*`u z8;xXiQ6ZTeMzGBkIM1sdxRpF=dZSmD z{CI_zN)YyWa925>`33+_Af9^tUalBEboqFspFB>AxM#o8Qhw!6qEx^?YzCh?>u5;Z z!5W0p;t+8E8)x}ksxqSnI+}4cjM)0;j z0EG>mW55i`&rXlH0V}CQW#Ukh4o(ELC9|{!W&8Dym9#J@7(7=pXn-Qw{`XC7AM_>J zKf(B7oDRTqwDbM4mRN>HFiZexotWoL)@tWNFmp)H`pJQ$+)L&R&=Gzg$g+3f`UeiM zSXb@ahCU6wY@PhCbs$c6a~^g_qhDX{Bhk(X>Gj*UFZl7-gO_guw7+Xr7;*#X`~?Qy z2+Yp8m0%b{dzMHlh_YZiL0Rq0GRuY5!-j_-K%jZbuV2nrK)x4==}@4weC?*n$Ctne zHGw-&a+|!AUw%+B&&Rkafb~%LmkuwyASt>Z;zZE%W4DQhZZ4qS>(|kt+;_ML<_FAl z8W0qUq!k^5{GgMzI7CcQ)# z96XI(jV~LNAD2#6{@w+GJjc4pG`^HpkyqES?U)L0s$<&?!nq7S{|n6#6od{+B!y1D zJ%5?AE_;zzL+uJ{8Z%6NYd+r3jX>nUI)DCyWPBPk64=Z9Zy1wPMhZ-uHlnAONHw(n zz<(njiT^ft{l6ku{=Zq8J`0OD{aLKRLBeF=642#Jf!(~=cl$H_frkWG03fLzc$yk3 z`R_u(m&z7-Q;qPg&-5qfpnHkkkf}o@>_QjW2v3JD8f_Zpe@>mN8!2H#9y3+aagsx# z{!ehtMg#F94qZB;1K*{yNU49IqMdW}`UeE3YA3!fR2)Bd|vP%(NUbx4MWU&W&42#^Wd)*SHyT3O`aLQzV`%WHv zCQs`hRlZ!RsmIJ)OHgQ{NcZ+FB);xpbDgP8#pTw2J6hR06DHq3q*;H&eaErXWptDa z<@v_{H}2Fh?O-YMZr_p*)rnb-Po05Y&Z^d7$l+*y33l9PKg@_9#ohvjnQ0TD&Sq}W zPGHfDnfOe4_w9Vo7@n~vUT_Fc!egKKa&kN-fz$Y%*>g?R*2Sw!XJ`Nk8uAix$*rm9-TS^ey4ZP^1_=~lChV7+s@rgY? zzUyI^xaJq8_Ee&~bH}zWl+L@0?yhfNa*YA%nX|)* zQ$EMwCwUBmou{>t1JPh`P$k0LUGj>FrwuSq`HT97F)Uemnq;(EYLhGZJwHJreOli$ zdDS-V?Z*R=sr)KGE}LT1&H+iW(OQFRvC($#eDNgKk)2=kd15*%!)}3#x_1bzq^A4& z5+qnijV^GlZFKpt>k=dgX4*-daj&8^4a@Soh49;!JUXyff9@5di9-9apO9`7hvL2` z|0p5qL3q%b2*cFIfTA}@dr|%d%f=Tkt!Lv&I6MS|tCzhC9*MV!PxC_!I zB}bjR-xaC}nv*@$KSbl5Inp!Of7QgPoxeC$`polIdviwdq$j~5{gFz5SbfrN>~2Dx zO`kaL*=~Z1=P|p{vkr+Ehhna?nc6%<2j{OL-~{X3q2H(I<`_whv(fbrO+V zDR1uw?(8MzmJ%F0KJ%3r_eSG3nMq8V7M|o%}Xy7Ef+hG5P#67y}Glw;1eqrGs@Jl z_os^A=IG|qYDU_!O)r*R63R|&&S4>7g_dj7mt4m!4b{L7$LX~YxzW(Ew)9V;Ih;y$Pe@X~T-dts{qj6N*mrYc>XA8O4@ zHdzD03;q$ilDfFBr}w5aN~YBI*J)eqBn}g5Z5!fxs)|t+pO1`7iFe-Ahj@0BVRE96 z|`;n#5IHTRO zzk1zEYVZ1;t}&%@HX}1X7>KdL+GRP*%WvQ}IHZWvaIg`yBk3p49U9_sFw8gCPdbw% zB$l<%-Z2j|y<|=P;p(@tf(xnCl>S_Pm9I%vv=sP5gdEen7&R7$|M7Q z`P4jfB5?va>zLCptoNDJ_Texs`0rA=igmATly$G{{9@Wa$x~^6Ctf9;jm01RAQK0_ ziC;!jyYss5S>i$MK;k#ePuv1pgk~a@^m(5MQ7b&>4X}*l-p$wz>H5F%9ke~$yPCHr z@iZmcdyUpiB98RC>oNOQ%l7%wA$=I=-MIZ!UBjZJVs(*QN2elo@64NsJ=+KaY?+o! z5N~RuqBJC|iW;eJEzdeU@+5fb-#0YBk~{8Ye&n$=+U+$fs%OJ*q<)+iEf(bT`sw0f zBq16y^21z^)xzk$X+=q6yshIpO4cW zc-W+SKOmxtVd(2wiHc1_hR3pw4b^Jb$Ag~8YpZ2RL#N?Ms~+orx+E3?yl2jTkKy}E zwwIa4$TU^U@#M%dr<1?XY4e`UtSMrxb)CmyVm*0oTFc7Z-Vrs@ZFBj>c@&?NN%AfS zoX48#Q_RiNDj)oMg1H?;bIo$Hv=tvlPxI$`ptAQjN`V3H9&&s+>)id&9M$@nHlK16 zq8Eu8BAFe>VOw*Y{Wd#OeuHO~3-?<-EsPGuJGlPhDluO>&T_%N;t{iJuyP!5ve7)^ zZK$FR0W|Ty#~CM2G5biRjc}+SU#`ZWZtE9CjCe zpH9@4{M0H%F=~C;d3WlW(HHWBH*M;TtD{3U65Z|>%K7>Yb{Q1a?ALO~iLK)H2J=hK z^v#vu=mR5(6#5WS42~T!Rx{Jq~Hj3$b?@p`i z`fdK5cqbAC`!&ZlOg)}A{egaNzn>7uSBq1MiKxtajK)S0}Pg7;Y%vX#V> zJ9(yR1@6mImdmlV3PgLj3xuk9s4}nGeh3@xH%s`(&X-`)K;0 zO0R3FGAy5hRD)pLt=zDRz8!`MiQt0Y;*Ok6H=X>WA*A{u5+v}M#eujm`OGDfYcJs6 z*Ycm=JuRWJ3MW343gkz=0kNv^wKHAfVwN3C7D*)LToSr$zp4#F>dURZSIH}!CjK#6T@&iq!24J>-yq!RE3gu8|V7ip)@<)=9!CNEB*-S uf6f9Co-2T=MZSgveaNN%H#zt9Gn~9Uv(^N8o?9sR+?Q3A$-8It;(q{#IB8G- literal 0 HcmV?d00001 diff --git a/qtfs/doc/figures/qtfs-arch.png b/qtfs/doc/figures/qtfs-arch.png new file mode 100644 index 0000000000000000000000000000000000000000..40fd7e28707642801ec0b984690a25c08e092ac4 GIT binary patch literal 49089 zcmeFZcT|&E-!>X-%&5qW4e5@6NSAJ?jwn??=^d2brI!%E$|y}jZ$ZSsP(ll#*FicY zLg+0hCDb4U2q7dncbsRQ_c`nR)>-HK=dAO6=j^pyxk>JOXaD-|y7rE59~o$|o#i?U z0)g0cwC@{%K&QJwppz3nodMoqDbHvIfv$se?*DEQnzK1)?`BJAQ9SgusoT%YY0SMJ z9Ow~uk;E1J^REXpKeJ+b|F|C}Epf*p!jJh1I{V=E1?KDB{GYm6m>0J$cidsVf?nVF|0nt1M<=sm-TfyyS2y|& zwp2Nfo}I$fT2=61CJ0!jC&#oTV)iZVLs61I+OtK zO`5Mwq1F;69~l+r1t^M=_m?IgIcJh@r#*+yft&Wnn*G0OPj{agK{css(3%27v`_)_ ziK7_z!PW6Af99rEPm)w}FSU3NNq*;(O?%$T;UBz9y#R{7QT>z$RF6ubj-aTI-c-w0 zz+2y+$Q1x)W8(abc^at2$^6+8^s84_<#GWbZTt$MOy|&KsXJ`?s!t_Q=3sg zhMb?-Adi;FBJ~Qfg2vxO60Z(`+pt#__;BbbD#l5Dht?BfnDN|oOKG? z>6|k9L$EUEYre}wrHTDg*a;A*P8SHfS#We@osesp1f522!f)nWIwj9Bcg@;`GFUJa!lc;Q=BRiP&{We3XwYJKr=IOhcS-$qrP zYHg8Pjzi4eX>(l(MUx7l{3>|U;eUm4d+pU zx5gf9tu=MsoEc~ z7IWW<#YR2-*3}n)&j_Zc7FXFtxUTPs`94%ZdCz>Vb)C-$EmHfQ8e&&wzuomVWM%aG zx6iS@&C-6(-}*muV@K5$!K1Qh?85|Oe#ngMC7cwvVBFQko*Mtw6`t9MSG?LbEFxTn?Z)~$Pg9IgXaR!OI z&rY*)!1J7E@yb#@*$U~{ht1L(b6I7KF2$SkVT)aFIf}-DP+)ja!MX9KQS!6m?LA#Q zc4TF@|Fg2M3aT9iKTvI4pw3=vqqmU0jpnf$Ps}I~rX)P)S8S-cZ#nY9{J!OiDbid{ zffgiPx&YtMT2t{?VhxcG{-ez4PsXEQMKe~Z5}01t4XtoxW6!bKGz{UwxY-9o&moy$ zqL-+Su5w<+>?{gNYTzl$&G_m+61H?tGN(LBA-#I}?R*VF;zz)|ows6;s-8E0$>ULh zI!)hc30rn3HX~Oxh#3z+V`NM+8>-x_U40EU?ehLmt?rp2D%F15(0PKs zXF$xEPQwUPj1ua05SYDIqda{7vQ4Hh5G|YGK_v-7@l7wt6Jbk#)ijhX?FzKW&LjJd zVlXS6!U;>gHI3mL*Z>g{Izy->#_~wH3;Q)$~T0FxZ_#f zUqKdT&C_$WF7Q3>$Wa^lvw_}XoR^ocVa%v@C?+XXR6XC@hh`8AcywH|3%5ZavJ5k( zV%+t%lP-a2>*Wt!;Hmk0{L|3nS^>mn#8b2PV+YzTTm2@h;|JPXV%GeY9;zhYcNUQ? z>Zv1`!3L)S-oedZDreI*G<4P$0s>w(V9Kr8QH0Pt)@hfn8lpBKR=r=UERi~ee~lGO z?H3*(@_DjxB_9FFZP&6?AM0_3{8|?oh42as&3{Y?Q|Ft9Sx20TwFnKsEVtq8k0a9* z({qcx*&g*xO*fDR!Je@ z;57C0T-zpCe!d9?K47t3v(TWDqlYjuDjrlZ*0BP3mF_-+F}S;4QzcqF7Fq!Dv867;oK z;u}aL!QO@RxK#<0JTH?te z!qlhRd;YV3*#rkWo4=7U0cQs|wyb&U!K%*l=pD3Gt9LfB58Y8X_=-pgR!grHTpmzt zczg8jz`AWr4-6@T=?aBCS$LbX+QNYc+BFxP?rBhcKpdNsL#xkE-nr-l(PTD+Cp6i2 z=LYBM6nrj5(s_f9Yk2leJ6zdU8fo`I{lI4b`i@WK62wXynNC_W4?&nj*`(&vd8X%* z!Mw&s_FLKRw9+2Tve!!Oss*wB+|ggas;xSh_K;5})fjUsr=zT!My3>~7F#g_C~E=R zthrYaz;ahoFQMrhsdSDsvKkBM*~CSl!wgA5#R|@Qw-{$j%zJ2?5odDgx<*Qe{WWZ7 z(o2VXAjCJ%WAp5NrTnz4h^i9hNRqaJ+>1>?9U+?3Ad1u3F}YVa%1&xR0X7)X@+n=J z$Rh@c3W24D6{RnZuT)nJyDV=u#0iPCMJg!gmC^>HyVo9T@r3G3giNtfKcm z)nrbiu62~AU`}^Lib5; z@)V*Xhn%h9lT(xT_bsZU!lQOED&~-%W#r^a<8>El8D!o+X<{#RgNLXmQ=Rb>NjS78 z8}Bwwjo2~J z)JKwA5k4;M%3+a0v?j>Ng!aXre$G{8vkF;Gr5aUL)syGS<9Nl`UH|kmG+z<%+MIP# zCxN$It||wMk!zxL8N|K4RAKXriK(pqUbjSeZB|L~?Q&D-UPF~M1>$C{JkWQPs&6dM zNRg|yN%jr@xGIR?o`mTDEP+K!!zO4O>M9gcrSxa2%yZ3O?C1Ou3c&g zu#%5UW!*ihqMz4KYY<~4tEbPO4p+shwE;=V4!o;vhmX9afv^SFqr?pNf-&AdODnvG z1Izbi$BuO1z{4{N+h9GVgVWpYjsEub{!0x`3;R49{3W&(CjI3!#kD9^;C>F6`AX)Z zbE5K=ohL~-U@#kxx1hz`M!mbm3oK&aRlccxq$4L1$WN`VH5+3#cVxs)x2}-}Z^#eA z3)c1}$+`5WWYlp8!fazIm(&7@>e{K(AhB(Zw!;VIHVq@2GN4=0G5nEt)Q^S} zvupv|w^#qv{gp3pxj!{x8-n)?<^)}CuZY|)bx8;8Ug^d3D`5NwDI2qp}W^zLJzoYtT78Z=f9lQ6V}sQb1a4>V^BX2UUtQM)Dzl7T(~ z*ReQC2RX0bF~E??Po32TG)m;-<{zl-@obyh+d2rlx5Fu1M4QR3UYpsv$7G`%&M;LA zjQ^sZ_O7vh$S7|~Rm9&(4ym{~wHFGf6EP~Os4!Wq%K=~R#C(*PYm}|J;y$U3;RVm~ z8upi%nCBmwvS+Am+tFMizfWV9h7@-PLMkLj&hY|syIlpoq$^+xdAzxdC(NiHy4yj# z9ATbe4B)ooCUepuJ3KdN9lN`m>AS+J4wbGqYX+DlQ-eFbmLnR7rw^95f zDL#vr@PiQFcozZ+kHtlWf?Z(|IX^g~g$o~3&w^f)=u@%{M~5M#(WUYnZqg+4wrrQ& z&@E)M-_5Y2BCLVWLc>l~(Jrk>QK%s+B#LHME)=#oHL*Kwm}juH7}cweR!@ze$|#E5 zwX&o38E2wFKZIanlde5t1!)|;W!GoDw6+%#8&DsQ3EAiiFVV2M+0N7OjG z0df4GMp_KO`?d4x16DXo$d4*P9;%UcvAfisw(MfD<|F%s&2@jFXuFb%bVkA&UTTCQ+tOv3K3Iu)4EWQpO9$o zNP5V;pVb|4qFr@kQQgjVK^w^+8%BK(RT!hbvxkx&MpAaME@*Qd|NTVDVenOE>Mc_O zd_*M5NNOPASJqtB_xZ@<_HA#H$FjrjnlO(lB&b!h278ouZ zEM;YcyeKt1b)F1lzFC)vUAna^X#?MuU5Y!HoUL&;Zl%m^6Q&-(*2wq>*|1>O-SMg8 zqroQhT?npFrL~<<-oh}$(2pL@d)v6;7Q>F3lu4tG+{rcr*6I+jtIIa2VU4+V!v|`Z z7NU9Ek?GOV5lql?ysYmeyULfPcYqdr3~pZ_Uk7Yj`h#AcjR)eGsqz0W6Vheg!7T0l zki)Mrq*8lY$&@qb5hz>H*C&`600Y7J&!0cvN5z7C^0*a!mZYL6dZ+==2hez#Wil&Y z=m+3-pu0?Ab}+#BuR=4ddz$FSpyV+!8Hb4sw;zQKN}fwcdwM?sB7TW=?uVFtwr3no2=Ns>1V_qsE+XCWO}^b#9?pWP z?2;~_a%)PJQoItSw#>2_asya$<-3yNxEw>uH*{QLq9L>aEx(l*?4nhik{0LmX*wxE zv)z`rZ>ONsJ*UO5*qjRtPwrgY$D+$e*?<|IvHt=zf2AL-z^TbE3pfour^&=VxRKsX zj%Vhj#X0Jk1)VQ-VB_URN?cmjO4ZxX^bA_$GKRynd~|wRJJ&L3V+qiyiN}^p0l;LJ zQ<<#wR*67QKwzkf`&R#RC~qci_!B}1L4<;v3u=`j%G12u^kvkrKo1VGxS8#H0#s+d zY$9>l5ORRwcxN~Os(ihjNhI%Yv8SaBnj_slS+qpJ%AMh1I`))acy2;RTn_?{4LI z!B8LtE)YKeeZjMAlK@(LV zi!TdwCY~1uSnGnN+jMV1L}=#%Db`%x|?h&>woO_hH!4!+7-D1H`i0BxoH{v z6CRd;IXKUr;1Y2dA<8?~NBJ68ByWpKtE&qPETnyN&{Xt502+HH#k9BnI4l7ggOe7x z?yEoX9J*FN2M%1bcE`GD21tx~$Ty@3a^qwOjflW)<(%ajHC7^T8)e9MHyeLk$V^Id zb0Mvy+z5g)-aVL8>>hJl<0j6`KhfS$zG1C@d!zu#$;Xd9{<^nSdlqRQ-89m&$8i?~ z@?cg3pX9Wfhr~DFEOya>L${4L*{@dXaA*_xi-4x;i?m*C9m6WQ~y z5ePH2dU)E1$JL*Ja=CS#`ABJ6g6h{oi4|vEBt(MvG0)kG{QX1aI<@TkyAH|&iL2V~)k&f9W4b~F zDalHzGg_uWdIh~+=B>h^oCf&1ufArFY{Xt@b%>#ou7plWagI{f*BaOD=fvX5CWHW_ zQF3MVd#H?BPDeo}uG<5!22o1x5+L9#=f7EnXLS%dg+Tg|?Bbg4%BLogC=1`vHr)K! zy=%>Av^=+>T9)t7PlL%OTK>nY-vu=xG2bf>!qJt*mwzLBJCpmc&l%C><( zrC>r@aZJghAt??#S?%@*%*ORY7&Cn`FaGs!6&QtzJ^6#~1n$%6 zx32xv++J0spN(rHY6eFh>rBIzP^I9V;UW>_Qz)p62WbM_2ikRbGa+nC;%ibTwAM=Y z#b?^nm;4PP{HbCm9hNPBkTsX>!Uk5(ON0j6N>=E>+LnLa`piQ7`N!#o4_~?!QvKba zs8W!FJOD_531>Tr*vXgd%dgosj4jQnx@)+Nu;oSWFepib5;bw`8c9w)dR~Uj_nz@+ zqwN*g9ERG4*ZOEX`yd-F?ZKr{NZ1&3)U{&+gpHe~#mrA$VQc`_p!GD_pT!T;m54wM-!0nkeO_xSRgfZA(Q2-~y(^4nQ@}%f>5Hg)J8-1^K-xz7 zdXE}Lz#gBomF@e4^GK$fYsRFXIApJHPMbG~0_+vEfonsjw6Ydq#C)f+=CT!?+-xN} z_~!TGRU#+_Jk3dhd4x7I$>lLaZdZJ7hu-$kPYcRYZRxlP-@)@;>Cd*hDza@X>6@Uk zYs?l3Xevrxf1_=v4V9)2ta+6^rEezhMFQV*rP+v*&sH1}uG*6K&`vNHcVb~iW}KG2 z``4SL1w=8#le*+#*AOYt*LdE0GNtOMI5!;+NE{EL>((A8mbwqPt|3QH&@o5y!4gIj zsTJh|Zp@BWQ1%v$K|eqZ%T3xnja9(W}LeikpmKxOyW38RJHn$Z*vP8~Uf(>Z9ZVu)lS8 zqdI@2P+ue4REu}#2?6B5a*$4%A2m1XNY1iNEwutF;7IYRHXWQSWNQed6&&sRV$m#S z*-U~7RNph&IWyavX6@!$ZR2gO-rrH-t}=}0_^pIBS-5Gy&DzUMgPuQKfc1$ zcq1EnRA6Xa!n%{X&+YH#>29W%a)r-x&fGlo3h546umvgD&!Z-u0f zOF;S#s#EK{bi$^i>Y)B$;d$zv7#ttRG~xMTPN&Dix_nq$_>*}v%~}y$*uUfkyiE?TZU^n&|?KVZU^)~ zekYtxS}#(B0jY%Z>lTd@@_nO0ZgHvmW^Vla64}*ZVANx^inR@%FPDJL%Q08*(#NK; z7C~K%6Oe|w6YB6w0A~a1KH3CAz|Qwxs}J;o_gvc6%z9g`4+f!*Z6;1Cwp!s2NiJWb zJhGpS$IR~+g<>fRJ(9A4`^Kpff-#_~)z%g@w#>tW+e=kiv@$db+s#&NjapRAvo z6eA?8)f-bTnWxXZf~+~BZIMfRS`(;^?i#82LpfI097xm|-I;2L#v&hJ^Nm_BDTyy< z5NiB}#}rNf#c;T_!!miO0xAZq@$Zvsa!o)p77ghmVb${{===8Bwr46ou;S{x1K#)x zuu7SZc%Ibx-~eZQWyeiSqzhW2T=e-u7oOiVuQ~d=;{3O2bt{cJF{X&O0V1|iK=AIb z$RDVi`U5{o&t!t0QPF7ha*sOQ52&}%LO_m7(=dj(7gj&H>JO;)PJ0rW3qk#Io!sewh8MH zo3>kzJJpXh{*yS?7Pw}Qa*1cgb-kF|kD~N{>+oIwkJDEArbwry2r!3M56W-*YhZd> zsMpT#KQKU#HLf)TFQXPBTwEyQueSVkI8Io{3OA1yp5v9!FK3<%Y`xQQr`@B(%zxbp zT(G#fFI7{BrVTUuRuKWPsmRZK=EgySJ&6~JcH|T!_a{{=u5Q}T)+<6TgcKpvI0`zo z0wh?gvC4pi?Y02HQ)wky5s{uP? z+)_k?n!&o`)*5D0l54DATa@2s@VlFb{3lovY5Dww`sDO{s_GoCd#0?5=XBG-?6hV9 zO7XV0x4t<69ypl!Wx4^kQJgyu$52+HgvKf$et|07WIjwyvF*2?LsChh zRq;{;<*8K4SZPOqjxK2+1|s=DN0LA4{zRxPTfByVF`W0d0xSs zgGS)}k(#kKr2F>zU_ye?%Fr>B_uv8CVdV|X_~Z(r!)g0@_%%x=Wx+vOo!4Bo&`0n^ z)?$Bh729VMnn97j&$97)=2N@-{7O@ka(H+!p;pW=Sh|yHEQL_X3V%PHRbH+YpenHs zVQO~Rine7t)$}!7&21`)7;YQyxP`j6DD?1Y0sxj5=TD-m3WsnxRb0ROdYT^)*Z#y0 z6!Y-%=)25PpYG4zL!wTstmT#FD8YAE50`S#A*xw6zI_SA=^T0ZgvnvcA*#fdxPtXu zX_|l^l(S6@Kh|A796n!EI!=!CfN`{~zz9KMKXeuX2Usd)C={v#llL57dd|w{=KtWC z>k%%Gp3=skZ3Em^^i57aK1>^JrfnI>j@W6mCke|q#;&cedjwm)|5VT5uSh-HE(+jK zZF7jDaL-QM1@@KI9b?X769RnJNVf-GUUReWaDOmBQTzD;a75)J!)&P% zgyVGcpvn5Yq9bO+NH>H$nv)u=VwHz_ZA~1k{7f)iSIg;SUB?PcROgQ-rPX|L#DC?1 z|N2hqD@-qUVp*laxf!7ApD7;AZAYeK)fp-}yA(nmA*79Jn=Ez$KrI&X5^JmT=_7Rp z%S#Jx-ecbn-QKWj$<9uz)wSsAX(`Rb$lg)P+U-q9aoJ1$bLr*qK}0EJTu5_(bLw%{ zc=*Sw_wuZszD^~fQ`;df5{7wRr)49hQ$2cAkP=p?Cmq2YB(HCi)xV*81J*wv zb)(6XUkiEys#;_rTw+OjbD6n$*ov%mwBTYV_-N$&2|^s<=;Dc)RPze&`A2mrL}(0Q z-mzFYhTRZ~wn8o)_xWq4CwJqAv-X_T=Std;)s9ezd?K(2lfBw+LMNNA=t zfGaGymE3aEzFX z4qlGI1g37Z%xs72N7yw?HA@o0>Yx#qmUmXt7o}I})Dr%ulEy8Aqy4V|f$) z%%r%H5V;Mi=Qz*Cw!A)8%3KOckLB#sr%!Kmc6OfgX+36oFS2glYPlr(41n#-`nuRf z*KQ594SgN;*j}36Tp2;|u6ghRs(FSnSE?=umktZWz7X@Y)}5O&UoS_l^70IYcFRWi z3nH@1yS?|i?5v_Tu8c9sMCGf>`F1m*l4d1q85L83&VmAK+FHXF=o{C#xcULDah&f?p60u37OxQioQtTm z(p2w_i0$^Y8mX>2Xo?RADS4>n>Ns{-oWnn#feCxj4U~J??uvaGeD;ALXIn&Jsk1?ZMA?B{Ns`BRq1C=UJXr zxV}1+NLRwoinA~`C-h;>>~HF=&CN6EBi_xUU&Q;a-vEmmeu`T1nWjwAqy5FP(ZVqO+!p}82?6jSp(EJ%MmZTLJNf97Lr zw7=-GxSSOE1;rXRJ5NC8)XK+>*1jSH2?KFD%NUTeE^35T}xi1d7YeWc>Ya15Evz(wCOI^)s-)2~b07b6?od;rC&l zwpK+*qn^8t5i;(RN>)wqR)zx6Bs+FvOi*GhE~n(v=v-17lgVt%DCk`L5$Oq-Yi;O_ zyNyLr`9?$N>C}K0tD!xB!)~#|KO|YH{b^#HRr^=HRfC1H0yAxCt!>hny{(Q24Q2zWg<*-&k2QzoNa36|ucrKCG% zB9@vXWk0wYs;cwQ(`traAIJb<3ZpsT?w#a5`04Q1<_5XCjUj>p1YZ{^7O&^+zx+=|VrR*#ZdUxG6 zAF52=aQh3fa zvWCyU4tRzMfWJJcLWpI>Zx+J5Z4YENH!boivX?KmL?XV)dBp< zWH7!Rpk%|yfz%%S#F#LuCaDgXQ9=S&U))c$5?!=7O!)%^X%|3DQ-3oqGg!bmGm<$eVYG_aid z`{z{L-77=c`?CR9gw2Y7!yXkth7Wvz8!Tdb|2}I5%zOI^prn5d~QkvXrit4klY!=z7L?#v`YEEr*EIW zCkh;F`Z1!5fB8Y$GXKapTWk`|My%@LGY$i%D}g?I@9*#bXTr0?G7c}r#KZ*tS=r~` zm?-zhXgA5w{MYqTUx8L);QqR43kjpFmzJ8Iz5xMFA;5YvF-u_y}wA-w1|y7?BrjMQ9phH+>q?ZW9Yy$`F=iJ_D?)**Eg+^69l?+*by>+ zD+t(7(vF1)fVAWNREmGF>`Z0e)c8kfAP3P;%$-k8$oDC4B)T9ZuDnK-Ev&GIop z5nn_+ogna}$3q+7sdLqaAj{tNGhWsGSO#*|*)o_0)i(-iGRAZ}kK3Bvtuw+{NTqC*)F79_ zo>0n#u_@`Jt0q{7^l&Sg{Z3tSgV_~o4h!Yq29=Hvme+bw2j_n7Y<6Kat;tSH=FCgf zB6n7b>zK938`dm#UD$*15~Dgq%{Uf(GVR8{Tr>L4Re2TTC&z2>zMPwm;hLnagOp-3 z|6jS#LaMMxHdAopdkdoSV%`Gu462V4t!H#flj#x;HYZxi2tZrgFi zJqhxVkvz1@Um+p=%^n;;nf5mssF{TS93sA~gGSnE87Pfaj#$p?bjeH2;U4#5Zu>PBlYsT zGQ|ipof+)!skW*wgGIcp(So6N4S`~uoX6L>9b4${~3y zY{qS>darxdXrCQWTpHmdzu-|dC}<%__}ga4wch=MeO}AL+G?zB03mzQh(PI0KQrJR zG8K?EQ+4=8a=pYlNv-s~Pk4~dRo-=mbi`Ti%x7p!;n+oSdn_ z%kw%gX+8)=)0fdQH*SV>;7Z4R%G?O-->0(rRyrpS5p)H1-XMdgNkUdYWF zYAEFOr_*7QkSp)3=M)tXQ#ZA2hK*yn8gpYQU*iu(`@>|y`8pfZpVqZRew~mcNEbZv^2dXw{>jPFh>}gE5vSvZ znx9-PS~t?_KW5LEHh3-6-02cUHpMBJ5=D<&jRp#((stwZ-N()91jk~7qHF~#RR!=i zBgwOT4Hx-hPsegdz_r@S6JnAT94;HqrK`*+WpAxeEd+J0lsr@(QtR3%8o5XrIo-?q zPZoY})QQ`N&G??uO&ss`*D!?0KQ=CEh1E*NYZe`y$&F>O4V@0$dLf)f2-iO=7N}+P zOqV8HI8uiCKnc@H3i$q6((z#b^OV@G0ywZ)S!SSdxH#dHs{)Ve2M=aV_{=U+(*R0*Trn9g<-}lito2??qx+$;DoXiG%0`maPMPYF0xyzYSUkthy`x0vHOood=jNb>&G9EU@=$vg$~90 zrdLf2<=#u^Sbh2Y`f}H;g45apT_zl{MdiVO?_B+m0Ze)OXUFdL%uD8h#qY25lH7sn z^W_R#PJR7f3Ng#vupR0#pur#iFF=UZ=0;xu?yxyg_xzs?gVnOXb`SKCK=((6|}YOMy5U}2psW}--O5n$+Brzj&U zZ3vpnNO0NWO9M-R8B*NwZK!%#cwV$0(k@7BSE8*Geb=}6 z-`iAbPb%3~etg?4zW8~MFNw3aYWv(i^1}Or|RORq*Z)k%ZjJb8M5je*J)3O0plKV@KhN{=FlM6h$ww8b44dl_sfs&F7&> zewL}WJu|LxZ~1<2dxJf?q+mTaUn}@tRavIwc~ey%C*aNHjfnM{VxOnbwNNEmhJ4eO z%N^GSzJb8Ka|sA2S5{(6q>p~g*!%C(S<-2zU6-;^MRXaEwj(KzHkC)SEaqmp-qzjI zv)|X9Iyh>xsCH0KA0o159aCmtpA7ljN80-K^2t>_XpyDItFlAuhy0|SOO4Uxw4zG5 z*PQLAsSo_?t%Tb7(Cenq?j(xErC0lg_?BWtzT-D)VTZ=kw5r}t)297Qvlum$t0&7r z!__O)2=7yrbyUdSw3x4lflft-^Q&$~DV^@~(jaQ9u0|rVqojg{toDH`!Wp6HtP5s0 zebgJ-2TtnGqgQ*M_Xq(lzL^r4A09*>a~-V=J#XziM-t(ETSHi6gDjR8Bd4L2_s%ig zjnt=&>ITuZe#cLbym?hZAHht_%nI>`@%USQv7qZMJ3D^pnwpeG&RZYZAigFSti*c?8Dk&SYQNwt~=2kDvem$N)Om1yTkP4HY zGTWk$-lRmzmD(TA3fQj}D2?)Ff2s7w7S)JdeovNfo2>xOhf-42D9y&KWJakSWLZCT zYJGn*01~vy9;w$B^ufSdrEb656XoA_@ylNcqN=2A^#Z8%34dy2aR{E4TI-eE9JQ@K zymH>L9$ayzRiT`pe!MlQq27kjr*-ibLCH5&b8^RKU-rJN4VRCiEWoJK z5Jmo8di=|hJVw-BKZ7n{8`&j5-fUY>w)YulqhHyTINdlz`gf zs#nWOWc$FcF{p6GRzuZnC-uXMqc(%%JT6+;=02)^)0lLGwdbB-hql?+9CRnDWSuv? z-%YZ(uo&tRchbW5YSe*Hn9r%v;S6`P!p9=CUWdrte#)Dh%a2suH1Dmpq+)CMk|fl z<8+d~lH5H0~4lt@~;M&y!eVHVI3z|EJYL*)6Ax zoZ6;AX^AND%&4(x>B-cChN(TF_dj1!Z@h#o(7PH@Aka{ zi1{w#=ii=m--!Jaq7SA%qFuQiiAi3`GbNZ}-Tf1>8 zzsD-EZW-N^{ANYLG{{w1gkmecUP_vE146yHuioM5>nyC6>mQo_LafH(u{F+r@rwOo z;axhBbof+IEXhPgHXk|M-0x2P4Z_SIws4qmxEJ6vd4ZC-oXUhTphw^L`gf0xo7{#S z=)Q$!*l0^vKg_JA;z)D-+Ue$$85ab9AlX1b}VJ zw1zzhAP?@6SkytZ#H!R?4j{M>HVja=d8ia+ON9q6g@L1WH99?v=U>Jy~8cgpjW zM*PzkKt+y|dN_%|88+?hMyrdodmi(YJ()((pV!|I7_R+%g>>%0IHy|aCvSq=je zbXGopweu-UiFFk`B0*H(m*d^`l$)40@^~EAxf*Pk{Ig)q1GvJzyp`bVWp4kqILPk_ ze6D5x>iAEzioCmF^L^E)cKnKt= z9ti~z&u)wZJ#+Zh`X7ph|Yxq9j|Q;Fb4f)upbL3PiL`1*YDi z_Ho?EBwa&db>Dq)XOe=g#@={+ORT^<=iD`dkAn$`v(+ina9y?_*LXc$=xTG~AErhj~~pub!|-&iy642?^TJFQ?YUqiyLgDTpjWK>v;?zIvSslUEG zRGBjs31))NCChL3Idw|R3D;kLV|i;L3K}ndQ^0B>q0=E^n{C%P+nZ^|&2dLXT&rX& zFr;`p!pZtkmWWPH%i)c7-?QYQX@dU)5APTkaroYVCa z4@(9#FT+d~Mx-#;dozi3=GrnPomx1}!|p=4nWaCnb-W&H)XDni?0A=?lAQKz-ULZ? z`9Kwq{r9d>2g94fl>QTcTs>KkW-&7-dv*&zljXJ;N6oDG!$=+Brp4+%^ShqZ2;E_# zi0Xo>3BOxUGOM6y<^3KpRYrgv+xbJpLG7fp*_dRZf=*Q2xh!+q2;QtLQeURfJ^*nL9b8k4Xs5^YM5>u^Jb$KV=2VeoR*r9>=(W)O(&*f$ga}FO1jK{T{nsWyHJmB7KrjO{M@oa3sjA zPJC|v6O$8w2#x=IzhXbo{t%E8^kDc0M{;zgX%y#;eRAUG%AF+L)9sC8-ve8g4!Bvm z3eZ)}UvPH%qoc(YpOAV5!^N!~7A;#}Z0=4-hCT(CPn1>O3%UHWr~$&#^>-2pR`L<$ z%pvD@@UB_^0`GRh;DUExVcD&DCIX%_&f?wapSYOa@e3EZ7q?^)VNgCZ)Khg?L$c=z zbWTb~NJh0F%=-0Z8*Li}Re?mLL6LcccoJQgv%cI^%lAlh$1?L=Q4TVlL1_JeJ!8;wOI%}z+|^&{PG4p31J-zh;%8cd}+8mf54xVU8ko{y5xmrN);dQeK#qSFp#(BAG`nTcN9@)zbQU%ix=!+ z$a~#pkgz#{bT8t%@!&V;1Y#u_Er$2uOF&%nS1RvT0xzWt=T|yfln;y~C103RQqHUt z4^%8$cQp&h8Rnex)V8aD#&R7@tc?$ek8k*x;)f6KeLS~&zP=_guYeh}?Ts1$nYNMP zEZwuS`G+E`;?9HhczGf5iH6v9{+~*BEwYcobNv;3q=E{n(}S#j5-_B$!NX0pj6wz* zi{v5>bd4I{XS^v&_V?F%ckAYvHvxF!`u{=Qdq*|3wf&+fDq9iQDotq?5a}Wvq-+%t zsnUDsy-4T;2}h2c&K-eo64YEl@4)7qGf;FkMm2%Zz@PCsSco z1ST7fUt86m&V_f;no$%5Ap7@E%5v_4a1{OWBxLP-qslgRifKUbEarRR?Uy<}kf0Bp zgzIVPvJF29>xPT696uQ*)aJaUb-Ez&I5O^W1iwB1$M>VY+unj5aad1fTJ6WQa)+Jv zwu;XErLiHO??U8rw0lKZz)aW*#(;s`g3&YaHC5cp{=N~OSV+pk-B=8?RC?_QrQRV; z3u-i<05w3w{J%x$(USdDmcmHPW=dB=}J3L@-qxeFHO zl02QtM{9bG_=Y)s$IN$ix_jhiguu+^8l~qH%8N~=+=lH=p_;C+9J!a82BGV!c0j_m zBGaI)nwrFE=qnXkFYI|%E06)YmACEDhu)tLK&`&_r zAR-Gf0H9n+7Dsc2UX6*3nDcUC$sgr8m6qfMzsKIC*|0I_U=nL-Chf|IUB_9KXycZ* zA7Aue6mw*8h$b#N6|0F7r6#Zio3D_R;>njg^2eXm$RIrFn)7-yv8LTz)jw~cTIuhU zEwr0*bWLGdW2NGIt(sxdnwgz|ub^6awjCpRde6oG4yts^@!3fKs%fzyse>Ye7z{me z3RwEaWX}KN%=_=PhUFgwWWB7<7IE4|mC=mEC{EnQG^&-as<-EyK?nmRrqOY+vt5IR zljQpwe@L(x30{|mN(px4i=IbabTQBozy4mfk@Md<^_?}jcrGjD16Q)Sh26x~?o~pp;FDiyE6W6hE{JJ79*T)_{;_Grfs-6m1t`rD z_aa%Frbr`@Y*x7j{IT!MfOi}Ce)8%{3Y!{HOq>gXMW)f0+?e%|un|KjtJoV!~#PKLX)B^N@!Xeah zqvb4`e#G;#qU^(p)7QKg3eD;8Ya;_DKDCc7Oo3R(KSfWrUaYiu7Yrm&awRuiqQb9_ z73m(LM2_u?(w~`)0uiKk@2QtdE!U*wn6BoqMll5U_zZR*WPb8hfF}Fp-ZiZq@chdF6Q{OTuXZ9tu+7x!cY6v(BG-U@ z+bk{yCQSo?oYd_)0OW|ZCnD8w!XPDZKXe!b#w+zLx&--H=o{M5}2pfo_Qg6Y#LLwKX9R$cHKV zcyog7FRD9Px-d;Z@wMRh5z~HD&?5CEg4G5;X4AW&-1(v}TH?y1O*v;BqzJ(D@VQ@X zJ(9QbCS8!&lU{Tck8zo7jhj`F%28vvz4#V$T&qS+9WpRY|LFD`R_so6vpD;f|@Lp12wF`;Sa3V(sQteEA|m+|-xqbVuxXNGP&(^LrSkk&G!J(tC8*(eA(*h;ydqwSf|0T z+e=2{uQ8unIWD(oi{1UwZr%Ybp_P~RU+a*ntNvJO2@J6PE5_z(h#hO$SMDA!8~D-u zDHM2O{Ym+YRbW5-2vyPbDkweRE*dTxEYPEkGEu3yRnD|%Nb~dspcYz2-wjEVdNzMO z$5987Y~X`XFFHmB(#3>Fr}G1N%^~?BWlIFuxrEyUix2<8alO*C_f1MI5+IM;)Jrsn z92IOdVBXH-msDR|Zep|W1<`3uz=DuX?V95h=8_~lrDl(?=ZU!7xo zAQ7Zl)U}YrtdG{%TA*xFx@iM-KjgJ<_5*s@yC|O&Y_+-`E3@>rBc=hwRFrfkeRV5&Qwv3 zY~E&o@%vogSj$ORi=wqR1#pocIkZ^+14p1?g_a2SxnaBbng;#gSK+su9<7bbOxo4* zDBMxNj(nO@U41`l%N{oAf_pg3vO2)@zobT-zYK!fEQ!shD$bXp50c7dWr7V z-$s_dO*VR4@AEO11%sW6lfI)Wi@lx-B?!uvK?DWKI@R}sY@Q*cR9Wb`L*umzqe~I)l(KxBH~c9lZdz|(&RA^{^}O&{|bNKX+EjD#}a2)qMCcN z1D^ovcHxxBy#1yngLiH_dU;pRrS{LFqb6BS14m@;fiGl9oL>en++)Srer)t&?4UU{ z!gDE3k>k##|7tdjwlzQft!%<9=93lf2dut|W`_3-UA=@X3m;Y}eZh*>f|AMl8zrQ9U z-C;qx8@mQibd4Y0G^y@Ns&RL*;~Jlu|-|L zmwj8H@C$shJO`AeKNO|aQ8D|$yQVW&Syc??VQe1amm~dKsmfWgS9vUPm(fT){c_qa zAifq4#$Ef?)GmCvv|jLhzp!u(&)LZ3(~)7~DFw%G^3_FOj#I7MT&5B;edQ$%h#Rl@c99hUp+W4{KVgn)z+BoY zpfx)zTmet!P03k>y<6XKj?Cyn}S|HykNQ_L=1|rdh-c+k(cq>dzKKhr{ z;B)@$j|aA@Ku}3vxSc0IAV{F;WcjlPq)z3cHwBy#wQqHzMF3W`{Quc0qwV2Q!R<6! zpix{{_=~##_Xr8eW~C8%21|)ssy>=6x~Kuc?4e4n>kkeW>Un>iqF-%B)xn}VJ_p}0 zSLdw>IqU6a``0*D^W~h8)bFmR;<0pd(svkmcQ{nG{KMa?v@{^hJN&b)C5FVqg!iknb{ERdV`m8ligH`4M2CGZpNI%hF%yNZsab z6OCPptt*1bO`QIyw3?LCbb{W1?PAK94ZB|chXb&KTziW8!!qps|Ee49zvo^1_q+K- ze5k1wFq<&f!xLkY7~n$sdEV5gC>`LZO^|G-l$@g|UL&@ViT2&OISd1r!WfWIW%>m{ zX+yh)`zC$6QFc7K22_rgRKS`_#6?Bovxe0=Q6Ru`9u59L?1 z4Gav{ZUb&61YpjCOPvm7?3az~X_hv?jC(Z(}m6fVx2hE+&T*C_ah*jhN%aB%OzY)?~ zPyxkueB6j^i{-O#r>2S%P^huN6H+y@I~lFnA02moZx8E`&iW)>d4n{EmT%qRI<`4^ zB#gWg`xNzeX)D2AyPdd*5%dv5L~#WzM}NfrzdQ@zdjGi+|Gix)lP_+ebq{TNM9k$P z6;iB>lh+n*25thDL&Bs@Y*M4d(k1L(b@jdAom?&fHpa z)Ff}$wDVGS(d#|HVLW{u?E})%MnfN$DJIEVcP$volWKqJgN+Y2X_j+ z{LliPzvw3_$=kq|tF1HR?TO>mIN~pC|G8fWF^z7k?J71bh3qt0O!%FT$yEw^$Re>Z z7(zM$@$zoG+vuFj9b5iy?p2M6Szuw=^3d^tk{pFB^+Z{ufI@J!ziR8PAmRm~RBP=7(LT zk`|b!Jb_wI{!Cs+7T)%zKzz39STzdRV*Fl0AS7TaUO6SV>)VVyqw_uTiuKLIYGT!Y2S0PE^vd&5;CR{ZLR%JvE@V6u;)@ zlfBESAjhLx>|UWU9b=dx{5MY+3(c#fFu4|ql138e)%!H_QY|^gdwGYrv67mJqSvQ6 zc&Vt?IsPfT%SR8D+vCu^%EP8a3k;jmFy=}EU{S5!^PTsZO=+TSts07l z`-pr7%-(u4CqqrgvonAPFVo2Yz+)yp>?N^~>p5xwxGuKHY~Ah75w0#vHXsnjV3Tk1 z0C^-Zb8W?^J3c(Q!lPq#?|Y-#6+jg^5GK(17I{=^WMY{*zW`X{;fo-4VAQg){!_M& zbyJT4t4z+Q^_hYvm?nAfz~a0S;^WOWUhY3urn|r3OKvy-AS6f0qO4&xDayq)30lHU zda8Aay2}J%#M1m%dRDwjLfV?nP^FW1aSh*SXMIV6G5wUWKG37h?xOY&c<9<#CcK8a z4MXjzsVrrqk}kZe(w;5FOSk}gpVkA}YZUvIbG_=w%cW)7?@I*k3fy@eN8~LDyDLg6 z<-k@6B;V@S7{Yp9zog4G;E2zD_*l#mtai&1*Zm%6S>%eyuutRO4Bm+a&NYul zn;eK&|}pxt@}VaK$=XY|S#Z9*Q++TJ6?HbzQv3(XNcF|XP>h?RSMY0ypLa3m|X zL}4dzVJkqM(5OM=>wRmTw!JpuFTGU-@E(t#Yb-l1`x{K=*BI`+tJPeJSnDqQPEue& zw7erPX8`M%+&{g=SnUBbZ>lTN;IPjU%EFN|eOHUw!3IL+e;-k7bH-vX=P<}aB~k-P zmUV!Zj1KlW`*1qFOTN{_X&g&)!v_UIzNJ5Wb;!W5-FW3X5MtWsK274R{Lrj1Fgzz~ z66av6Ls~WiYlQ2TMk5(NV3xn8B`x?j`6juYrTWzWyEFALuNT?;C_b8lX4O+5-5$T= z*C#&H!YK<_hd#wH{Hdv`MhAoB$I=a~TjHztIGtnjW)6@T-x6W>=8E!;FB!NZfgPpR z10>C#OcdN$#KU0N05h4o=>Wt+Srjj3yFCvdAR>i=uG9zmWJQU9|Bi}40<5bcHcc{3{`S%VBnfN#&i5QY8u#dKS3K*kMY}w6&?+-jBz583tPsjRF5?(>;jB6*py;R zFi!($_b}CfmkA3KKeaOto22}?SZblvP2R?`)~_d=-U9L$Rduz7N64cD&f>N|R@Hqm zZwoZ0if3lroFf_cgM3V9_vly!ohLyl%JCe)95n@4lIL?n)HG&JCFQpgAM8SuQqCPQ zJ~~ZxUF3IrSYiyui*p;)GKbBk{l45*FG^@9(?vtXkE2j|6{B;Mkk)vL{}^GtiIew1 z?K+C@D?x6(*q@mxlWldV5WdW=QkBPlT5LJVMRvTY4(!{o<_+|14o=Lk3Pk#Omgd%p+4keRo1|5;>IZPSXou;suT*SQ zp)YNIwJfhk#T0*}sLizf@Ra};)M1L+@oV=%N6D3mp8)%C<=EqsM=9ekD#Gg}D06|D zGKYI1foq1Hpb8J6qjy-t^>uNp&VPlwAIFWVH@JPKOpN#jZ6^69(>W7+n4Y#kgIn*WRy|D0;twKe zU8in;9X>J;_y^!AO<1cN>Q~eidV?g&0~UD^jL(XH#GDFX*ErY7`F!*=l@eOifd69W zgRn%{#Kun4{bF9a?#nH_rTQmGv6O9(v<3~T(DcBRk3!bRxpvlnKoV*-Jg`A#6}+YF zs;$)$34x2PZcRoW!^=m0wQ|*+rgB^p0trz&f-`x0mvd1KWssfxu9t_j$HCa-9mL=@ z(txj%hoc1zZBXIe$jTt8n~LEV2fL*lQtJ5oU`@O}OPZ~+TGQj&Ojq{@|4oue{d||@ zd(iiNFS5Ilv-o@tT$;prVt+iVnz-f%Lucf|T$s8}0mjR&e;QWwQ1beO%)F>dtZw)< zq3Wj>GyrSHVkH17-G8L^^Z08i{RKd+PPmHe&A9JVLTEk8#4A+8D6k}F@2 zZE9^?f~;M5EnWqLNUrp)WdA47OLP$)CBrGZM)Qayiz@7}fTWYhicj#~AS8ja5F;is zuD7PeE@^`Ta&;d3^vP@<^7bsv0!$gNuekro3<%60Z|j|y;O$v|OaX9nl-0mdHphj< z>>9oBqh!6prd1`|%9Nv0ihEPyQk49?6YSiF^SH%l&>mi39?$k!{e2S9albcSMx2DH zfQts8KF2v&>T#-8>vwNOJ?w%=xnD?{Yv;{5*%V`ZF zrf=D(Pi_lU2X^jH{H-Rq@8m}7G+@lsVxCnPs!jg`3GyCN^1||8g~Ym3w4#%%`x=nt z_0Ny|H&apm#bE-;F%&}f*Y%%&2xSh19uXI=XswKKk)EC4BOhk9qV%{rO2W@8T+6dH z`_ik~mHI4hpq*fKQ6dLbsM%I3M`Fd0>!EnTpYzubrCyVT1Ut8M{zY!&Jm*vhD)hqy zH;fHXA<8o4{EACUOUDMfdnu=k0yqhh@7|gA697i5^IXUN&vpqb!?(h|&VJl*DVwQv zO62F~e~P+D^@O6g1p*Qg@Sk^oWKW469QC*P;VdlbYc4Gi(X$in>f%8}EZc>_=f?x&{W1j~_G68Md_WetYk8A|w5vd}WO4Qs~HRphz?zWibi4 z0mqye$tcv&zETj%0Px*mL#V%2s7yPc<@~zdGyfYi@S7v0P0F>n2wcizF$&}D^2PB^Bs&)WzHyfVsekSzj zuS*Sg;Y5&#H_FLg0>mh$FeN>x9}7Z=L4vw6*8sp789`xmtU z$DPJOKYvP}M(rf&NKqT5puxxHz$}v}_bENQzM6I_B5cp$x?kAg{n@}Xp<}FFiNDT) z1qPAjJ&GoEos#Uxd@L-ooO{_wb0>{YKwuQ$lGKpbs6GXDuwJB!=-Zh0S8xJy?Jm=Y zvmB%0Ua((0ge5l8LJXjpbd;5q-`L+9N-MX|=?}_u*h`W0P--Y(% z1j%!3Uo+fEsO;Wtk-pO1JPo<5{iqEMp=urwh9}s3a1}9$Su2}(8A$|PDi@+!|8u_b z@{jG)Mm(Ka3H~CdBBFo==E4`>9n-a-t)WH#dxuZ7IU9J1bV#lgsX(#aRnEG$eIfVG zds9;%b47MnigX&N*`6fr2J@`S##lrSLqQ}jj-UsD#5&yixX1B}&Jf9z#r1 z6)EqX63|%8xa(^d+E23Iq)X52w@?|pt%7l*)k1Fqxdu6*#kDv;1vVniCBNP6gOjwEbD$w=wyFtnX?eyUw3 z-_IR}LBE>lx;5T*i{bu(u}l_D(_C+j+!%sqi1*1pDrIl?(j$r&WEaX+eepxAU)BF2 zU(w+%vxJg**DNi&fo2Cp5M%%xTJHK+AKTFe%la>VVfi z_3cTk_geMqeDQogJh;(iz3#)FY(8^P`stiqXO)90MlEW7mUM>gEU@$_&wX$AncgW7 zC~Ppdxj^=9aOFb)1?{_lBR!NCSj`kSy{FKv_~8|Zl$;fPCWh>}L&OIWvk96rQprf$ z=wRaVG-`Kw4XMlHr1`n3ef+1(U-ILK)-Qy%iiYf?kMtn~wrZ@ndt-SNS+63*6qrvN zjW2DWIfkIbEN7w@2THeYA1P1~tXSWwwI|42)4SSea5J)=i{xDAPWl=u>Zm4KI7Obp zXghSoy;jM81(G-yGgU7qy$cJJFzHsy*8brjMev9ve_-Du>v3LPPCqp!>vLRyj<}eW zeOP7|Srf36*v|GnU2CV1(BbZ8xFNB#)&MPrO}EwU)Eytjbv}mjtL+4~;zTVPa-qe3 zI%VyTfVq>ef;&Pf7VRG7R%6YSmCWX*Pgl|}c(B`-rYyeC|9k!}UqjVoAV;_zL}9=5 z(`)OcDu=fiHEy(4WfeflD-C}Ge+Guy!iBknJejlZ&}cDX&Hppd zwqqhZ>hl*EWXIJzT`m_Z&WQ`!GL8Ie%PiJ~r`nmQ8NMseRg^~-@zX^!(i~&5*NW<& z$ay(7U{@N(;70kRY{EXTX|23TFj+u|&*OU)Vqg^*x{)+8?UR<)T!ihbljO)*k72wPTdl3mG->niNIR=E#323y7%8(=M*Mr*0Peet6<1fVIemnnw*~yHzv;M)kKSY|KD;`U{t2!?64|7&DaL;GPlq>^|D`6H+_jz98S)c9Cm^&Bq??#XxRg1J_=# zgYaZQHs46o@n$r}Zx(txJiJ0Z1Eth+hQx$K87RGhX>E z_hlbX=0N>yJuukS$O=?;BOPz_Q@y1}3p(buu$%$ck%={s>@=@0nw zA2sVW+*<)(%pCZcEFbHGJiVB?g06SPD#R_*n=7t}9I1dUk0XrUR86o^ z86l0dq2Z=n?Q$S@5qDt(`I1QA5C=(KhqNh>_)2S+Sp-r3)aTg6ee6t-J_32z@c}ns zrO7#+WSmptlKhKQ~KyNxlGc)cd4KV^l-)SFIb5(nhfu!fur+TkL;i|` z$>nlCPz$*oS@7IUSM=G2>bXo9uh;(fPQ8ki=a~ZYp4I-&0YPu{h+bxcsEY^G zBWwQ%wJLpRJH4sFj8Zs6)_u)9vYOau%OHvwXIwP7Ux<^C;u9(Gf*QtfX|$2OQDe;0 z?63Bi@2+d`0sT!1;y2_>W=xD67{BkjWD{PU2%-OJd^+C>=*f7(Z2|nTziQBe$jFIm0WmWLwoxD-r-LZz4?8-0#`yI(D2S1{Sw8gu-ol zfjRqaSvM75nYVA|TE1lUcKs|AiSWKSG4BKvvXG_=Z_Zksfj;(;?x;_SS$pvj0->E!PC{41!i_ zc9%N$@DSu>zIrssIwZ66oSyv}$q?4|D4I;p7Q_izYNmPvvu=00h#jbgSKed~P(j3JrYRPV~GI`LOvl_ib~cFP-s1HKiV{ zt$QMSuUP)%b8W6?S9ovxuEX&0XdNyV3?{&8b`c^=UmNzwrjYf{XYS{P$&C?)W*a0= zLN+Xh)Qm^Yvn*BLXrLN=gzUd6LLz-~gMDR@Wm7x;BY8G*nBLF%MIL#=EO7ZLyWLE= z$8j)ov?0>CN|cb2=SSZW>^ARebO)$Rd8#WXf{3(M$nH%IC)ArhIjIlu@)VImBc@cy zq2LQ0T~?aj++&f6)!r>e*{4n`*mekRCaL^0bXWnA%)9GgWPNYC%%`M!Bf)m%CV!Z_ zVa~(m5pdYEy%_d*fi=JG zawvY+~?-)NASx*IH8=3+o|C^9U%%yl=;xx zVUKm#JTnnt%iKjj`gGHnF`MV^?MRQ)qbpCE+DNwGyQ@vs<8Ov#1Pno`^#!~%B;}Ae zn=)$JOz)s(p_K9!Q}NZ7b+7U9AzR$MdCxHKC>WRcFspulZ79a*>e2qZpDl&~DsNbZ zb;~7w{~EAsS-rnj_b}n*+lF=0ON3{ofSZXb|u7${m{i%`^IL;ZWf609(d&W=`UZl=7k87kLg&$>RnvtfOIa ztmK|!yr&-&2DMeexdrA56?V9Kafi=nMaQlxBH?N0|1OfopYcNb% zFh`uOX*XDKfPt(m3ytFzDX8)B$U+^z613Q+4c@Nfk?{x#sfGT9CJmJX?-;5IufZuE z`(`@{~3cV9W+%Se#zjkk924V0P}BCP#=WU^qtmc^mH=24?ZI*+0KIE~aIbqXDa z(D&dK8o1-vf}@?SVUCuoTJUN?_JD21_bh5)PI@nfit;BHLQ*z42U85{YJhnzdZLN7JDGgoZr@ zoqo^g=Nq#d&3i9`+Hp`2_9)UjbBo+U{Nl&VLF{G1v5)K+dq8;1G0oh^3n>N*rO$lL zjgY`v+9u>Avp@G~i^~a>?$BOym!<4lD+(-gFo(%+YYX{tOnE7r1G*)^F3Qn$lxTsI zcYu-=$v0IFht?UgdKygjmZnHBa&QSDwI8D5BkYemjxVVou^4;K=G?yEZ+=^?#&v*L zv;XbG(O1y%(2+T)S!EfN)yJlg%#_PL)mj1PT3AsNSsHJfWCnq^E3rh77YX;7#VgOp z149N4z90f=TKk^&ewwh85A?o?(nPxfM-~0@0_Yt=Tz*3Afx-pM%%CNyAz@QxNcg1p zl9lSLj2awYnIwQttr0F2;%7B)zMtT6V6 z0g7tpxRy7__dndl@7r2v0T=ug)zkI*8F#e-riNs>s0)7FG`rZok9BM+?RA?*ru0GB zO>abqqzi2;p+dbWyG;x<#A_d%kw0=U7VPBe%D#ULoZ94pVD4<4Sgs#kY?auT7lV?I zG9a69yO=+EhnBZpp9d~j!}@-eB|!0kU2w9tbU=$t=cxkai8eVf9%U1BNiAX9z;1C( zz-tg*BQGkG@DZX>%EW zbJm7lkw8Ls==8EI76SFE@%x-~%CLa!>vivoUVz7`{v4UKNVox2{*Rh77hrekfoGKU zrp+j(+iBtYYOp}uG7k%>^Y2M2GH#U=m>3-VmbZqat&ujD$wHsCD^Ifnv09bm(Wl@4 zf5ewx zXI(KVTLG>Tz?1a9_L$zQi}X}?TG#CQMuGlbq^DlWqYbHVLf&BPhh$D!3b#5~7VHPT zp3|5hd?Q}6^!~R|RWMXi(Uz713V|c7K4tiz1D5T8F}z5hd%dCs|1{)io-0M}BLC#k z!M4{Xd^uU};(MPEz*VzYH(?>*$9fm0u0p@t$>y=}?d=sFw<|xqwTl8;{yF{Q-|DLi zcy@qG`&rcg`NsJVwei0mvls;iOzUVIK;f>gtnvG?avNKiXJQ&jyL@%CL4&0_88xZ+ z0}P4e&9QmMk5#V`=oT;HF@X)}7kj3O%_+O$3EyR}rKEdYi6W<>G&w8o(!=s&%FBal z?Tz7IulKZOhQpArw`FRcdJft@otTXoGxKzTf6e*gqu>LAKMXbQs)P8ZuGOIqG6M4Z zyB5@_aud7K|9O+uh1}ocP!u>nSX|C??%Ss!oPU>G_4C;;_Z~mIaW1g($-IZ{2o7UZV5oZ;2WdOmDT(Sep(uv?2Xrj;afEp${je|ZZSzoQ$` zQfStFI1C1hiq|guynOM!(J{oH93*w?bj|3`pnk4a2gL){Pdg<$_uKBpUZZq-Xhi<+ zLj7->9D{d&Lxiua55||5i!06~le$ut>lrFE&EXGUUKj~pnPn&MiaE)>FFjm=)!r`C|q!BB;fd((p~%gJmW-E?6aF;{_@cJMx=QlJ%9*VI4U zPk(r(P!L?Tsxuv=F5HRa?Yfp3@?sU}+J3M;+-iUhK0D&EFbbI|AkZ4GUm+$+B#E9b zb|0u%ik!xl<;v}m9TcS6>hlP{URJ+R6noE?BGM9 z!EZTZzS!pip{FKcy8iywL9^JV-(G@IT2e!Zr~oWjQCF-^A(gG`kLSJpi`H+0T<;yc zH;WqW=wHh0G3)8OjS=yh-_ObD7p0HmUhM@9sqace>gBsn$>b`w;a>;2kI!-U%+@aj zeQ?=fHp%wClkL3TJ$q7{60fg0BG#(awTJQt`_r$H^+5|>DZ(W^)!aQY0ofwfRG~qx zEAwX;Y`~&a@9&3shH{r_-OnN|(px7RH{nN0H;DnqHJI*rt9C{3j}Pc!4dbktbC0E> z3;2Cp$~hfrsVv3uRq>sDT?^J!cO@>&D|M`sbe?VAvCmaLnCb{tF&vm1rp*L8 zz)kC&?M^kw8N#f_=UV)$o5%h-O$9r3Flp4;d-8q#C03fw2~u>>?7G_&^nTCWB7Ctr z34VhHA67cprBX^9ZdlPW?fP_-NEsweB?d6}(q^&dW_(zMVhX8?XmH-7MY_Rk_XSe> zJ2nFq;0W^C{2y7r`^81{@R+p|MY6g+tbE=F0FiKPCXU#(I7<8pI*L@0@CHEVS zUwE@)A(lN!6dJ@h09{m1kMvWSj-d}m<9$H=X-ut(epsRrz#Wk0S_HOZLk>IGWhRrn z&bf(ws{+PC9nGj`EiMGlGj3dWc*PiQz2gbK(q;7K!QsM_CWg8tIB6(vekH7^J=m=p z7Ym>u0b{V2dkf)EkI(`AQA9H;v%ng;0c+dU`Ci4u%5t}PU^>Y4$V@OkRwE^~yW{c! z>uA}}7O~Vj73q%@2p?#{{*Npq)i1#vobZEao2%_0dj(u6KX86~X1_cDlqZ|o-t4$>8v4;8gXuJQ0v%IYemouz zLN@I#j|y?XE89t%<@a|8U0zT%uQsXa-CKH#uj>q#A;czpTZutIx%2mwN@|8hUAN_h zdN!y8SA{H79|@wh03k^2)R`x-A<%JbBS|7b2U{CMc;xPEuhB-)UEb~17b1T}fqmEqsFPIM-%F5YB;@1jA)x*qpX^PwQUI=$WeS`P_l}w1>^r6{|fNHJddDKO^#6d&mHEG9BfRk?BoI{bDVsk?l># zZj)g4{;O8Q(nwu)cQF?_zMh*mQI7*6P23E=4nxb6B359#dwa|swD;>@&hh85h2m4k z4<;w!MmMk9SA2 z{S27=gYcS5H-VtR#yPlLC_Duu>sYI;zlI39-XO?nH^a+0L@w=JM(8p0do%(|0erht z-#FP8+V?QP@n>c_3eycUl}*s6JMf1R^~Vpw($JkM>7#*vZ+a}yYI(z~uxlScK>kI% zqN|!ad~2+`Fn=7^!)qv{32v#ZS8ZxKw{&1;peyf)?yb78#4~r4q%qOfJnFgAJBA4i zE+!_-7c)&bVAlk~-TgOagGgJp<0?}0Q=7#lC5hhoHpgFO2)-cTF!uJWXNaR~&+*~r z#v$X$NPnXA45ii5vee7Tu85u3?=l|x$>X~r0$fc));8>>AA)s&ZaVm>;-QJ-os2Zz zDPPx3$~O*JXW5=Lt|YGm z)DG-uXhMHGvcf{{o|E|&`o>$Y30;PT{e(S-smhrLC1?T}{b+whOeVFp{>8f6M`GDwz?~=4I`&25)wq4~3u6j{#J`{PIEB7X2str>LkulBhc7CtCT7uv0#D000 z6i)p%gwj6eDC4im@)_<;9C1(K9V`WurnMWZP;8MI%P_s!Jv+#*SsggM(i4cPUQ|_4 z85vGONQ@V$`L#iBUL_%h)`Eht74%NfMw}m3YqTa3-&2GWBUu~>c3jdDFI^u?Ll;K~Sl&eLC^oNncM&!o@b8ZIt0{n(Y3qU4%C%#5v^iCXB29ZHZ#MMA(Mc*cr$PQEWAoZ5DN_ z{!XLTQ-`?_0Fm)XOImKek3N{(2Zl5pawd%KWNj~ zNHJ-1-%?Ts*vD$U7iqw14qlszA@{@OIEP@e33h#kcBp*Z)oTCIJ*^l?TI6*gt~3Vo zf947zIt*(rtUAOl|J+(&V9;YQMmjy2i8NgFa#7*>{Ou0sfPy@`)Ut~kFPryoa2 zDo{KqQ+b)W5~7xD!?TR@v6_{teAevf$Df0B z9$eYgH9a3g_g>8&q#*fBShCP_qWO7E*j(bL3F|hyX2iB3Cuy5u z7j@3~d%ZGq(Wn%!O#=H-x;M`%c~DH%&HU6YEzJ%oF#R`$aL&OO4|QWbYlAhBy!jF# zib->mp>RjvMVt9W>sK?r^4Y?pDUm#bT%#qE>@HyV(8BiM2T{!oNHV{afYzprr(%!1 zOe%qjligK5_3CQRa+wls&1zf)g^K4 z*XF;$9g4s;;Ki&f1WE^`1056-BhJAZvRX1oOy+&a>|AV-YRVj27kq53-B;uOo&6o` zG2DBhJQ(w2JqO0II-Omh2XlY9vW5`f*=(5K3bN9!zxRE2t3JpXL%!h#YZd_cOWFkW zi18P&Qj;)#^j)lK!!Y2(k8*S_okSNEP;;UJR9%_CB(+!05B>R>oz+YH3GR>N7jA&0 z)w+j$I`IPws8Mf8SM~BZ>$CkpsoblMO>%C_?NVV`zzchqdTNn-KjVUZxd0hSthTJ^ zoAk%%(}#Iuo$D|icDB1I{bm+tyRaa3rI22h?5!=i&n%ZI(K8*Y9J<*ghKr%mdMZ*XzppB$fl+_l}Bs9M~#{r!sh zL6`)LX#EgCW~dujht_P}^ZmaLHt>wPk#^?Lo649SK#1cmZe9 zYVqd2dK7V$5fq~kmki-z-TAf#VAW31Sl9|H?+kAp6w^R&f-5T+&mFLmL5tE$S|~CF zxS}w8*M%(&(&-M2p1~*I)GylDq(4=V)cB{XQfXA!zVGddn%RW#)VHkn4H#AW)kp)NqN2holgqJ^c+w;RODnjo=@f%(N zwZyuXWjwlS8zwV+m*lh=_`%~$ z%BM8h!7>sl`VJilOmOS1mn|c~+~y=La|&1u(I=0;nH=hk$fq4SEW0K1{lLrwy0eQk z`Y2*m#;aA()H&p}epaFEw`IP~Wf%V=P&TpfVg57kPx2gxvR?2t+t*HEsCCbqd`d*2Lh^ywz3!k4VvH0TdS2hV;IfQhvIX03yT+ZLA0&_An6S$S3! zTt-BhXLtz#GYrS=N2t#%#H{hmTV;S&;K#i5QNM#@{%?}rSQSUl3u1kq-N(j)m4!&C z)X%I`DGv+uZhb>iq22rDrvX@DsDdQiY~v*(L_QlHR66NG^j@%5Eds*&o{og6#Mpss zMMa0~;N83#exJ5SPK=3u!acUzO`iCo%gb57u1mX~q7T2W0^aB%HF&B+(Z-X2oQPI+ z>X}arkO!_s`qOvHX55>FxXO$THt#+Gdo0h2OG?roK>&l5V_Ge7Vv4OS`ae;W_=AL4o`be7VpP(}G5zy|hlbiP9#ryL731pUA+(%bYF zi!@sHy7iB52OKNPq?wYPRpDRnnD7lwsTWz)-_O)S8E)P!e9m&@>O-Lm9m*}5!xLi( zX(h{Elg&^$cwS_~p1Xc-pTgN}!fN@&Xmpp7d_;BYaXdz$6L$_62;e zK*a*(EsOwd`I0h1DBT198fv@@ylCW^1q53M4uAG;RE8^ktu9zNV`!9*MQ%Py3_K{a z{4TFd3hjT3S2N>|swovL@#(<4M*iu3SwGaCkp;1b4Zhyi)yr`q(R*T#46z7aCm;!} z#}=E~RH3SIu>)=t;^#fE6_*?#Pz;NuVi1P_+4X)gys4j}-qM>p2#?mEN`q=YG2R zl04^}wX@3J>$mn^d!4<@=kx)C!Il*kCTfE})~{gR!2m*j!!Ok?tt*fG9{Wb;M6t56 zAm{~jYK|nE2TLkfn*VrajC0CNxiC~ffxZAgxHtH4Y-e#ja8k%*IgSV{4{OX2gsU@A zyg_|&dv;66;MZdU3d+R3{(7)*eZ?vJ$*nhl8C)T|rOq2V2P(qnZ4Cxhuf91fc3%it ziIw5G^#Gy~FM%LP)!(5AUMqjNGC*Vniw52LWGSWU% zHcBWAZu>gcC+q5$$*Gc#v#JA@f6rcTcr)(3|4$RWY6{Y1^2P*4AI`{;Uj)kOv~C>A z)McrPdNSK)rj{q<5+3wsFMU!^T4~P1v>M%i4g#efQNqxGWibbXFkmV4g6Lbphn(2a ze}XqSYfyuuPevdY0(7A2S2HViw&}dxy9Lzw(;6qA?VV^Z*I;}rK$)K!^e~?4UEc|} z(-I!t+oS{>I2YUd6@8n-nr*r`eQ%nH4M!Yw)tHi_`!r? z=Q9WS$h_xPAdR4&ZkAf3EXXEr@5Fh^er=;t>|a(mYt3V6t8%5=LAXcA;E>d(TiU8c9mx+5`Lzj3eC2GQeI z5x}m)2|!$1KwKw4T^bWsnw#uuS2_P=SKHo157FBRdZ`2CS^S+({rJ0cR+<+|v0`a? zKfAfr+sC0^=IOV8aBEj;agwj?_$$c&88Vog@^s7jDS+F7Od5B+%C+H^sU_`P*k`$w zG3<(!CCxXu@X)JalGt{JKba|X%3foGcHQFsjrf{^*cfP_A?tbH9(SWE{d*fohI2Ux zyHjercvtN0QA=P?)siaqr{w{Ow~1&RyJDSzP#P$M+mU|s zm>+aguQa6E8~krPaD*I72Z7=NowP4H0BzZ5>;6zBsJN#Se zUZ@C&wV(8}65~IfVf^px*Z(z9={3ZNE>rOgh^{GNBMYZ`3H&Q0or~g^ENEx~sm>FV zD{mn^+2~az-e=~r`bFcpw{7#DH7u;K`y6_CCwV_`dJU+38Gz1{*ZRrKzO^w}zf%P> zcbJC_+z4T`g}5oAS#HOl0cFiVF#@39{gd<1)^usqGS4t#eezUY*z9uu3KPZWJrO5Z z+Yzbq3AYGsj#^F9#9+$6s7_;e|b1YV3J-;%TrqTNQmMsFomB|FV<;O1@?6)ec+aH z;pY2CZ4N~(fj~+&mVcT$?*h_Y?GMB%5iQM`XndLPqJ<1VMDN$02F_s7-!=jWoT9x+ zqBW8zfDzqaBF+VStpc-TjaMo{mV;3nBDlu~(i|R&N8RQWP-&~Z2N$#zKo>gVj_I1X_~c}rcAo;M{ZGKT zUurb-+AoUP=sRU18d@J!npIf%=-w={;8HgrMZ;=hiT?x~hnFKTaxGo4(!d1`S6~sn zyykm?#H7^St!K~qm=eM4T&KW^_0GBQ)-7IAw_0|9>@36&IigocFTmS(eShjQ?B8*( z)cwLAXEW!J+u9F_uVjF?gW^&E!}zDG3k4Ru4ru~#o{VeYXF16Q;J1qBN000%W-c6n z1glv3PEe*Ti@&(ImIf>Wun&RXz}-4&{mFAq6Txc)w|-mp7ql;5aKD;HN3k->p|#=1elr{2@Z- zHB5(1N(DHg_;8fEy%Hpi=2G4-utur!w5rCBYzccu*okf&KO#2t0pjC%xdots#*KqZ zlLIVtb~9KoS&^4VSZAG?lnNU1K=DQv3sexsZR(PW1jDwd@Ae6YH}_(&wm+N?(7R^3 zKBDibj<|Z9H;sY(_4C$#r&62L`snByKE|^Ty=vz~%t3brxa?8lj_9LPlpAMOVprB* z6_F#Kq!VX7+SI_boscV-LED&`S0^m3eWvj`(y>-=pVfwxDGizTbKM)+&snwg*uZ?e zw-X*-W+Xi3 z9vJf@cLh0ml9eboV>Omf)wxy&#NbDu>#bJi0v8TTheOdQm)=EZe$P8thz$xNzY*VV zOGe{_yr}Yx7_&Uh8njql1`2YT9yj3p#ji|-_0*gX>t|N(&!ITA>W`%nt4W8edgF~N z0_Y!-smgV`!9!G|lJ{GP5gl1-uIj_&cLggh=Ni$jdSn?`$NSYIY-GV`nm6GqoYN^l?oHpd#7;~ciztj zhZr4j3UbQUtGJYn@b7QB;|kex)l;dKduh&_D;s{uwlU=55W&xVTHaIkTUo-Wi`ilZ0fU$(lS~~_my$f>%rDm$Ngdxm;APC;*SiLg2|J@@nNAjzT`}i93sk;2+3td;n5_zHHGNbhUA?7W4G`pj-6Qbs6^@s&7K;G@Z0Ck8+nk7`iRaX@6JoB~v7_IQA~VE0@x82TFAU z+wE$iK5Jv*_2fSIcF=(f-pG{bp;V^x9`%nmB^I6xtz11G6E z3NI+a>{8;dgG1QEzHjl>lI+~(G3X3$%s(b5=o%)X;8lyE+o z;BjjZQQ;2iBBW@k2@2qkDx{1X6(cNmvBp@8Z@jt!jQy5CCox45;{yXwx z+uk7W*6&DaYJuQO0;Cd;Zb}xM6J1DIg#}Q@!6HNqj~MK#Uvu6>JFtcTf52bPZwHfy z8d{f@gY=;jwJzL8oVmZB9kSZ>jXhuMJ4QS}8E)GXXdA01AdWb`SCTkAq5wf#XXMnC zjC%t(8GjtA+9$r!_!boBIsxnv(r^KskvR5j3oAJ#o*5OWRJE?8-hQMDapf!`%WK?4s) z_}XiR<=5&IgFcYQ8q`KcZT%9&0Yi;fptM$?qFEe8I9fF^KxwY=;ms(W z!2r83de6Sdl6-A5l}E+Q73dW|(wYz^?DQ0cpuS&2G``&e@=6xVfR16d4FdPm!TUj( zy8F9SG;$^8+30oYEMPwPqM#y6K-4CxS%RxtDPSt9Ql>{`xE|yev&M@$a_Xhmb#S<5 zU-oM7$1B*~2f^f;pdh?X{gPUI)cm)HD?>9j-LDu}bDN}^8&07TtEKXA^u;CKRr#9y zJ3ESB8``c9t-N4;Qr|E&S*uv%y|d=5G;^O3grk^ADM`@FqpCS=heZv6QKLC-7%)os z$Hh>PthqPx(58}Ozm=IkCdH9k&wGT~R+iDT1OnDMYKqwXtDk)x^Z! zPmS+_0v+36_Sf{e#ML`svarWzYFvYOH1+-n+&X9m-`fe%uvq;w3}OFzKKxL&7ngtG z0F|^adzyD!h`(^;+KWBHwjzNBaI!}z&@CVrxs z%=_73%Zb%Hl%eZlMAM3lNMotqacyh%b9i2_t=e4g4~>E8B{eloN=CIBTSgb}J2f2{ zuV4YS_8A?6S%$^TA*aot8dEgVt7oCR-dj3iVPN?bz(r9%ro4JY&*aZ2_CB$Fm=LXg z3(cnu>6Bm*_2E)$q#bEYsN^7H@WlThfTy4X2lf~_MU6zd4ryD2? z%uG#9okHxH90mLJVJBxLB%!skK$40`?1vMKeAdnsW<6^Wji;gi#pXxl-wjGKjr+E~ z2>AGXwBx`ekrDOJTtEn<&cPQ3ldq(uB_51dyGnti_gz$N#?9pA287Hp2}dw^_(8nX z*PGReH6p+sSL=`aRPMX>FC6c_FVM}}P|n??uoyxHjtpo%^LeMOQr8xa_g$)tS?H17 z$f3{RLayfDmpR56r)O4xF)>&-TJj*R-p{-!Oe~HU^)k|Xy{`&xOY41BWAox~4G#j2 ztv>KNOpTc80>jS7ZbnMQkDg$yCVR{9P+q5!RZ8ncy{trHHZvv!zkdDx-<#V-HI5&E z+(5p`%5&^|JpW(0x@d2Ym>GVs`}~=e>#oHP;;0{FC$noJKeKde<3GzE_rDWgiktdd z!{phNTJo8Mq=Lhnn^kS}AI@*Jak+cldwsve=9W~iqLT?SuZATS`u`aDI)@6i`1L3z z>V*m3aA%XWeWSv^4-Kj#Kw-V&IL%l;Ru`ilaw#RJ#mhVESavhOD-F>$AGUj{ajNXe z7cnpG34h6Aw@WPaf`f!|Sudh?ZC*W3IhkO8cWcz=Rmm?R7CWTRO@AZn-ll*KdH|Zg zJa2;kEHN))r%1m5E1$^6X{#pA0H~#@I{B>dW^Mq5ZeZcFsaT_v2``ejuAJKSr8t#1 z%V<(_%chZmCBYNEOzX!+B@h^Btyh7EP#fIZ1Zw+9 z&7_;fAun@}&}_nII~U2{2X`vO&{w-OZ-`|AJ)D=*C7k!D`=sFa?B>GjKM1P4-zDb% z-kn#q=AcO5xKK&Kt5n!xoM?=k82N*^wTD49F#DkF2%UnKC};%#@?AIidgU?Q!n!)$ z7`%I`Oa#o5_bCHWMQX}@blcr22E7^2Bg}v%6u` z=kWqouq(9MRxE;fv4+Qv4T%Ny7DQO1Syruj+noNwsIBMaJ?hj?D@`t6sk`~5mMPgm zx7}ewc|o{Wtwo6br)qYv6f=cf%WDfa!%`-7w{eEywC7GvPz0;mf|?p{OK~7CttDcd zFa_QgLD%X>lJ+@4Bj9u(A!+hvZaBE7D4trdu|m-BrTujSC49J7b<_j-AsKcXh`RP2 ziqYKAakh@&Tz_s$Tdgl8bq&@5hvb#?ZT7l+gZ%L=&NWXNtp(#pY^V$MqVPtu5)GNj zFH~Q4M$@6dk2$sn^$S{yOlbJsv{?rWBeKh3#mExv(yHD}Bc=Xh9KhL`)t52>%e z$nrU=F6B+>kva|-S2ou{>{hglNjcCDZO|Ck?_O@INMcbrn3^b~r(d=K;8|CPkk2#~ z34$FqV_`4lhoPB3RO3y{tJTx>voM2cAr(2RFv_q4#%!tF+TGoKlw%_wq2o91>bxUB zt(S1DB`iVdS~ZXz!WXu({{Fqz^%ruHm^lXnps9UExaMm3k%8=NMt~jA!So&teb5gi zge|R(hs_ISDP+ivXnXttt;9K&&ul=DZ*R1wjHfxyGlqhj!!ul0l zO~6u}!2$5HnA=QlXbu}F7JDX##{FI1U3ZEm-hc1*=;3iJrHrEkw=YYQQ~*SndRKkr^G3Jaj5TTs`wg;-j%BfCuqxm)WiD9Id5P--odonhMD#(jPrX*(#(*_62V z-*tzQLqG_Qa*q(mJS8V9RE)7cG^xs6HL8gS%f_p?0aKsSL6h`B8Bus;W6`l@Ja^XZ zea#Ut03=u4YH)>`ZuL6YXyV~I4CX`*ycy?*h@g|lB2uW-_+VAt+Z1iSj(ZtW=%JQKmiRi(}OcJcI&nrn;PTm55GB9 z@sOx0_G;|`*4~Hg_5lGs@1t%@=cgk{2rhBdy&q>`Dl3*JCDTg6PFgk5b!ols*pn}o zj;WKB@QNQevgjoHhy2W;{wM1z{`1@*chDo)=s{@2 zxvN|at7&4cGz(8c7K*15z-PaWH*yDZ8X?r-WuTRpkdx{|i)^$L_m zVUw3EZsXEAx6v(@{ap;po+0<O*0qLT*KG=$;DqnUU#dp|)N$&D9m{Te9y15-O9SNDiKbFm0WRiN^8New zgN+9Q9>VWx%27mqV!@2_3Vxby3S=Z52u z1mm6pD)CBLM&InULPu+OG4@d|><_sRsaY%Wydci)_XOs>^GHKQ-}5GNzH5$Mq_=TUX*3`$@LCF_f``J(M}tB7$`T zn<~r3Ha3rq`eeG=*9&}NW#-?9sbC3`l=ORLH#b z*Gtl?87V7tOd!L;wfg=sw(OF@hV|Zv^RR7c(aiO^Vn5A4=Z?;}Qjv0F+%{j^ss)|w z0b8AINvAJfOINp_I{!C!VoIgAG zRWZ}Yc@A2yucU2=SsR%{uZ8xn>j5SLjmv<$JUgW3n?vWoJwYwA_?glZSqVF)bXbqz z9y(k*vyY5`+a9Hbbt87VslO5NIreTj1;&!GsRcuQ<}KMa&lVQ`(SGW!Pv59p{NuvB zZw{v;HF(i~>~!6^AeYpH8bn#Dp3K@ENbl;@q4c_Q6(>^5u_ITbYEuGJ-cB6tHHgVq zP9kOZ7mPN4yjIjBd}`^)%-lHaV99X>k0|qn9sF0z6{CP;+`aoJpgvW~P*p;=>6bh%<9b5F0 z7l+iG%8BnBNw<)J6U~i#Z^B2jeIe>6dQ-dE^|@owV8eztG0Tx3Z6$&)z)xKFF@F=^?6TGMBRmmI9;UcXs8`@n;+_(UFFZ34WNZSK=ya7AD3h;>*P zer8RfX$iiGOtjwxovpmGoG(4sn0?#8oh@KvS-^E7UB;obaD>*|!3|l3 zZi!ME{Csxy?6a`Q!HT3SYOph{&-3#wwW9Mrr$+@EZ`{9s==w6PQTFe)ed)Ck5#&l2 z5=yyY+@d*U)ZBQa_b?5IJ}L0@P3eEN1lFC=WPBX&UmZ24!Ud;7rcBy)i`50U))4M; z^F(H(PAEzCFsnPyuYXtO6l&312w<3`R#Y{FzSlh}N2ReTz5B9x z)lAu6?!7oh2zA7AM`D>&Bw=p!P&yB?-^P6EdO2>iALtTq*@+b5Q~`rISm*Bi2O%Oh z#Fxrk`)&!@32qrao#cV;cN|};7X#}j81fcVf*_4=T|O&jXHaR~3H5!wSga!640BzO zn8)ua z$oTs0G`ldA0aTLi18h>r%_dx+vg*e@s*B7FWp`%9vu~ z%cm=X>&h%o$c4-E)()!I)I{9*Zloxs&R;c&eKBEe?B&6_nez1(ymj09kl=lWDvV>qlKZJ=_EIjM}F!DN=YEK z8SF}f=P7(L;CLDvKd6z7z!}Q=Y2}{eRCo|!m(=0|^mb$oXGGaiPQapK%@M?X->uBwZ_l5(Hitq!(wanM*XRHNhXd#>PW0_7Z2WB zGTSGtZ(-sH2_4I3yWd5{biToC)G-Rrx1NlrjH7ZC?rMm&Bb|KZwDrr*z7fS6{rfX+ zj@wS$+<4+XZFxKiSFN@=8TsV>dxHA&IHc8K52?<1Z(ulc8w{AdW?5(hw*OJq=3#-` zj>w{S_Q@wpzUwa>qV|S}ETypeterRYFjK9_v@6!9#UV=j+rsW;1p|($a6}r?GbZ{f z-4<@#g^pUDTVXPVSj8VJ4Kbz~s7J!pW>F?{jc&g{Ct{h1Q`Wqej<&B5A+K#gnz`1$ zrY0fg{ge6j3Gp{5?TO;QKYpAG66d)0XKwNIz0Fk-kk!SXxv|n;e$?`wJkYT@X9N`< n{aMlK{NJtpZzKN^oh*og=!6f+SeF#Bkt{FT{aJaz>)!tXz%XaA literal 0 HcmV?d00001 diff --git a/qtfs/doc/overview.md b/qtfs/doc/overview.md new file mode 100644 index 0000000..518deb0 --- /dev/null +++ b/qtfs/doc/overview.md @@ -0,0 +1,11 @@ +# 容器管理面DPU无感卸载指南 + +本文档介绍基于openEuler操作系统的容器管理面DPU无感卸载功能特性及安装部署方法，该特性可以通过操作系统提供的统一抽象层，屏蔽容器管理面跨主机资源访问的差异，实现容器管理面业务无感卸载到DPU上。 + +本文档适用于使用openEuler系统并希望了解和使用操作系统内核及容器的社区开发者、开源爱好者以及相关合作伙伴。使用人员需要具备以下经验和技能： + +- 熟悉Linux基本操作 + +- 熟悉linux内核文件系统相关基础机制 + +- 对kubernetes和docker有一定了解，熟悉docker及kubernetes部署及使用 \ No newline at end of file diff --git a/qtfs/doc/public_sys-resources/icon-note.gif b/qtfs/doc/public_sys-resources/icon-note.gif new file mode 100644 index 0000000000000000000000000000000000000000..6314297e45c1de184204098efd4814d6dc8b1cda GIT binary patch literal 394 zcmZ?wbhEHblx7fPSjxcg=ii?@_wH=jwxy=7CMGH-B`L+l$wfv=#>UF#$gv|VY%C^b zCQFtrnKN(Bo_%|sJbO}7RAORe!otL&qo<>yq_Sq+8Xqqo5h0P3w3Lvb5E(g{p01vl zxR@)KuDH0l^z`+-dH3eaw=XqSH7aTIx{kzVBN;X&hha0dQSgWuiw0NWUvMRmkD|> literal 0 HcmV?d00001 diff --git a/qtfs/doc/qtfs共享文件系统架构及使用手册.md b/qtfs/doc/qtfs共享文件系统架构及使用手册.md new file mode 100644 index 0000000..8088f48 --- /dev/null +++ b/qtfs/doc/qtfs共享文件系统架构及使用手册.md @@ -0,0 +1,69 @@ +# qtfs + +## 介绍 + +qtfs是一个共享文件系统项目，可部署在host-dpu的硬件架构上，也可以部署在2台服务器之间。以客户端服务器的模式工作，使客户端能通过qtfs访问服务端的指定文件系统，得到本地文件访问一致的体验。 + +qtfs的特性： + ++ 支持挂载点传播； + ++ 支持proc、sys、cgroup等特殊文件系统的共享； + ++ 支持远程文件读写的共享； + ++ 支持在客户端对服务端的文件系统进行远程挂载； + ++ 支持特殊文件的定制化处理； + ++ 支持远端fifo、unix-socket等，并且支持epoll，使客户端和服务端像本地通信一样使用这些文件； + ++ 支持基于host-dpu架构通过PCIe协议底层通信，性能大大优于网络； + ++ 支持内核模块形式开发，无需对内核进行侵入式修改。 + +## 软件架构 + +软件大体框架图： + +![qtfs-arch](./figures/qtfs-arch.png) + +## 安装教程 + +目录说明： + ++ **qtfs**: 客户端内核模块相关代码，直接在该目录下编译客户端ko。 + ++ **qtfs_server**: 服务端内核模块相关代码，直接在该目录下编译服务端ko和相关程序。 + ++ **qtinfo**: 诊断工具，支持查询文件系统的工作状态以及修改log级别等。 + ++ **demo**、**test**、**doc**: 测试程序、演示程序以及项目资料等。 + ++ 根目录: 客户端与服务端通用的公共模块代码。 + +首先找两台服务器（或虚拟机）配置内核编译环境： + + 1. 要求内核版本在5.10或更高版本。 +  2. 安装内核开发包：yum install kernel-devel。 + +服务端安装： + + 1. cd qtfs_server + 2. make clean && make + 3. insmod qtfs_server.ko qtfs_server_ip=x.x.x.x qtfs_server_port=12345 qtfs_log_level=WARN + 4. ./engine 4096 16 + +客户端安装： + + 1. cd qtfs + 2. make clean && make + 3. insmod qtfs.ko qtfs_server_ip=x.x.x.x qtfs_server_port=12345 qtfs_log_level=WARN + +## 使用说明 + +安装完成后，客户端通过挂载把服务端的文件系统让客户端可见，例如： + + mount -t qtfs / /root/mnt/ + +客户端进入"/root/mnt"后便可查看到server端的所有文件，以及对其进行相关操作。 diff --git a/qtfs/doc/容器管理面无感卸载.md b/qtfs/doc/容器管理面无感卸载.md new file mode 100644 index 0000000..2e4be2f --- /dev/null +++ b/qtfs/doc/容器管理面无感卸载.md @@ -0,0 +1,31 @@ +# 容器管理面无感卸载介绍 + +## 概述 + +在数据中心及云场景下，随着摩尔定律失效，通用处理单元CPU算力增长速率放缓，而同时网络IO类速率及性能不断攀升，二者增长速率差异形成的剪刀差，即当前通用处理器的处理能力无法跟上网络、磁盘等IO处理的需求。传统数据中心下越来越多的通用CPU算力被IO及管理面等占用，这部分资源损耗称之为数据中心税（Data-center Tax）。据AWS统计，数据中心税可能占据数据中心算力的30%以上，部分场景下甚至可能更多。 + +DPU的出现就是为了将这部分算力资源从主机CPU上解放出来，通过将管理面、网络、存储、安全等能力卸载到专有的处理器芯片（DPU）上进行处理加速，达成降本增效的结果。目前主流云厂商如AWS、阿里云、华为云都通过自研芯片完成管理面及相关数据面的卸载，达成数据中心计算资源100%售卖给客户。 + +管理面进程卸载到DPU可以通过对组件源码进行拆分达成，将源码根据功能逻辑拆分成独立运行的两部分，分别运行在主机和DPU，达成组件卸载的目的。但是这种做法有以下问题：一是影响组件的软件兼容性，组件后续版本升级和维护需要自己维护相关patch，带来一定的维护工作量；二是卸载工作无法被其他组件继承，后续组件卸载后仍需要进行代码逻辑分析和拆分等工作。为解决上述问题，本方案提出DPU的无感卸载，通过OS提供的抽象层，屏蔽应用在主机和DPU间跨主机访问的差异，让业务进程近似0改动达成卸载到DPU运行的目标，且这部分工作属于操作系统通用层，与上层业务无关，其他业务进行DPU卸载时也可以继承。 + +## 架构介绍 + +#### 容器管理面DPU无感卸载架构 + +**图1**容器管理面DPU无感卸载架构 + +![offload-arch](./figures/offload-arch.png) + +如图1所示，容器管理面卸载后，dockerd、kubelet等管理进程运行在DPU侧，容器进程本身运行在HOST，进程之间的交互关系由系统层提供对应的能力来保证： + +* 通信层：DPU和主机之间可能通过PCIe或网络进行通信，需要基于底层物理连接提供通信接口层，为上层业务提供通信接口。 + +* 内核共享文件系统qtfs：容器管理面组件kubelet、dockerd与容器进程之间的主要交互通过文件系统进行；管理面工具需要为容器进程准备rootfs、volume等数据面路径；还需要在运行时通过proc文件系统、cgroup文件系统等控制和监控容器进程的资源及状态。共享文件系统的详细介绍参考[共享文件系统介绍](qtfs共享文件系统架构及使用手册.md) + +* 用户态卸载环境：用户态需要使用qtfs为容器管理面准备卸载后的运行时环境，将主机的容器管理及运行时相关目录远程挂载到DPU；另外由于需要挂载proc、sys、cgroup等系统管理文件系统，为防止对DPU原生系统功能的破坏，上述挂载动作都在chroot环境内完成。另外管理面（运行于DPU）和容器进程（运行于主机）之间仍存在调用关系，需要通过远程二进制执行工具（rexec）提供对应功能。 + +容器管理面无感卸载的操作步骤可参考[部署指导文档](./无感卸载部署指导.md) + +> ![](./public_sys-resources/icon-note.gif)**说明**： +> +> 上述操作指导涉及对容器管理面组件的少量改动和rexec工具修改，这些修改基于指定版本，其他版本可基于实际执行环境做适配修改。文档中提供的patch仅供验证指导使用，不具备实际商用的条件 \ No newline at end of file diff --git a/qtfs/doc/无感卸载部署指导.md b/qtfs/doc/无感卸载部署指导.md new file mode 100644 index 0000000..c15eed9 --- /dev/null +++ b/qtfs/doc/无感卸载部署指导.md @@ -0,0 +1,166 @@ + +# 容器管理面无感卸载部署指导 + +> ![](./public_sys-resources/icon-note.gif)**说明**： +> +> 本指导涉及对容器管理面组件的少量改动和rexec工具修改，这些修改基于指定版本，其他版本可基于实际执行环境做适配修改。文档中提供的patch仅供验证指导使用，不具备实际商用的条件。 + +> ![](./public_sys-resources/icon-note.gif)**说明**： +> +> 当前共享文件系统之间通信通过网络完成，可通过网络互连的两台物理机器或VM模拟验证。 +> +> 建议用户验证前先搭建可正常使用的kubernetes集群和容器运行环境，针对其中单个节点的管理面进程进行卸载验证，卸载环境（DPU）可选择一台具备网络连接的物理机或VM。 + +## 简介 + +容器管理面，即kubernetes、dockerd、containerd、isulad等容器的管理工具，而容器管理面卸载，即是将容器管理面卸载到与容器所在机器（以下称为HOST）之外的另一台机器（当前场景下是指DPU，一个具备独立运行环境的硬件集合）上运行。 + +我们使用共享文件系统qtfs将HOST上与容器运行相关的目录挂载到DPU上，使得容器管理面工具（运行在DPU）可以访问到这些目录，并为容器（运行在HOST）准备运行所需要的环境，此处，因为需要挂载远端的proc和sys等特殊文件系统，所以，我们创建了一个专门的rootfs以作为kubernetes、dockerd的运行环境（以下称为`/another_rootfs`）。 + +并且通过rexec执行容器的拉起、删除等操作，使得可以将容器管理面和容器分离在不同的两台机器上，远程对容器进行管理。 + +## 相关组件补丁介绍 + +#### rexec介绍 + +rexec是一个用go语言开发的远程执行工具，基于docker/libchan下的[rexec](https://github.com/docker/libchan/tree/master/examples/rexec)示例工具改造而成，实现远程调用远端二进制的功能，为方便使用在rexec中增加了环境变量传递和监控原进程退出等能力。 + +rexec工具的具体使用方式为在服务器端用`CMD_NET_ADDR=tcp://0.0.0.0:<端口号> rexec_server`的方式拉起rexec服务进程，然后在客户端用`CMD_NET_ADDR=tcp://<服务端ip>:<端口号> rexec [要执行的指令] `的方式启动，便可以调用rexec_server执行需要执行的指令，并等待指令执行结果返回。 + +#### dockerd相关改动介绍 + +对dockerd的改动基于18.09版本。 + +在containerd中，暂时注释掉了通过hook调用libnetwork-setkey的部分，此处不影响容器的拉起。并且，为了docker load的正常使用，注释掉了在mounter_linux.go 中mount函数中一处错误的返回。 + +最后，因为在容器管理面的运行环境中，将`/proc`挂在了服务端的proc文件系统，而本地的proc文件系统则挂载在了`/local_proc`，所以，dockerd以及containerd中的对`/proc/self/xxx`或者`/proc/getpid()/xxx`或者相关的文件系统访问的部分，我们统统将`/proc`改为了`/local_proc`。 + +#### containerd相关改动介绍 + +对于containerd的改动基于containerd-1.2-rc.1版本。 + +在获取mountinfo时，因为`/proc/self/mountinfo`只能获取到dockerd本身在本地的mountinfo，而无法获取到服务端的mountinfo，所以，将其改为了`/proc/1/mountinfo`，使其通过获取服务端1号进程mountinfo的方式得到服务端的mountinfo。 + +在contaienrd-shim中，将与containerd通信的unix socket改为了用tcp通信，containerd通过`SHIM_HOST`环境变量获取containerd-shim所运行环境的ip，即服务端ip。用shim的哈希值计算出一个端口号，并以此作为通信的端口，来拉起containerd-shim. + +并且，将原来的通过系统调用给contaienr-shim发信号的方式，改为了通过远程调用kill指令的方式向shim发信号，确保了docker杀死容器的行为可以正确的执行。 + +#### kubernetes相关改动介绍 + +kubelet暂不需要功能性改动，可能会遇到容器QoS管理器首次设置失败的错误，该错误不影响后续Pods拉起流程，暂时忽略该报错。 + +## 容器管理面卸载操作指南 + +在服务器端和客户端，都要拉起rexec_server。服务器端拉起rexec_server，主要是用于客户端创建容器时用rexec拉起containerd-shim，而客户端拉起rexec_server，则是为了执行containerd-shim对dockerd和containerd的调用。 + +#### 服务器端 + +创建容器管理面所需要的文件夹，然后插入qtfs_server.ko，并拉起engine进程。 + +此外在服务器端，还需要创建rexec脚本/usr/bin/dockerd. + +``` shell +#!/bin/bash +CMD_NET_ADDR=tcp://<客户端ip>: rexec /usr/bin/dockerd $* +``` + +#### 客户端 + +需要准备一个rootfs，作为dockerd与containerd的运行环境，通过如下的脚本，将dockerd、containerd所需要的服务端目录挂载到客户端。并且，需要确保在以下脚本中被挂载的远程目录在服务端和客户端都存在。 + +``` shell +#!/bin/bash +mkdir -p /another_rootfs/var/run/docker/containerd +iptables -t nat -N DOCKER +echo "---------insmod qtfs ko----------" +insmod /YOUR/QTFS/PATH/qtfs.ko qtfs_server_ip=<服务端ip> qtfs_log_level=INFO + +# chroot环境内的proc使用DPU的proc共享文件系统替换，需要将本机真实proc文件系统挂载到local_proc下使用 +mount -t proc proc /another_rootfs/local_proc/ + +# 将chroot内环境与外部环境bind，方便进行配置和运行 +mount --bind /var/run/ /another_rootfs/var/run/ +mount --bind /var/lib/ /another_rootfs/var/lib/ +mount --bind /etc /another_rootfs/etc + +mkdir -p /another_rootfs/var/lib/isulad + +# 在chroot环境内创建并挂载dev、sys和cgroup文件系统 +mount -t devtmpfs devtmpfs /another_rootfs/dev/ +mount -t sysfs sysfs /another_rootfs/sys +mkdir -p /another_rootfs/sys/fs/cgroup +mount -t tmpfs tmpfs /another_rootfs/sys/fs/cgroup +list="perf_event freezer files net_cls,net_prio hugetlb pids rdma cpu,cpuacct memory devices blkio cpuset" +for i in $list +do + echo $i + mkdir -p /another_rootfs/sys/fs/cgroup/$i + mount -t cgroup cgroup -o rw,nosuid,nodev,noexec,relatime,$i /another_rootfs/sys/fs/cgroup/$i +done + +## common system dir +mount -t qtfs -o proc /proc /another_rootfs/proc +echo "proc" +mount -t qtfs /sys /another_rootfs/sys +echo "cgroup" + +# 挂载容器管理面所需要的共享目录 +mount -t qtfs /var/lib/docker/containers /another_rootfs/var/lib/docker/containers +mount -t qtfs /var/lib/docker/containerd /another_rootfs/var/lib/docker/containerd +mount -t qtfs /var/lib/docker/overlay2 /another_rootfs/var/lib/docker/overlay2 +mount -t qtfs /var/lib/docker/image /another_rootfs/var/lib/docker/image +mount -t qtfs /var/lib/docker/tmp /another_rootfs/var/lib/docker/tmp +mkdir -p /another_rootfs/run/containerd/io.containerd.runtime.v1.linux/ +mount -t qtfs /run/containerd/io.containerd.runtime.v1.linux/ /another_rootfs/run/containerd/io.containerd.runtime.v1.linux/ +mkdir -p /another_rootfs/var/run/docker/containerd +mount -t qtfs /var/run/docker/containerd /another_rootfs/var/run/docker/containerd +mount -t qtfs /var/lib/kubelet/pods /another_rootfs/var/lib/kubelet/pods +``` + +在/another_rootfs中，需要创建以下脚本，用来支持部分跨主机操作。 + +* /another_rootfs/usr/local/bin/containerd-shim + +``` shell +#!/bin/bash +CMD_NET_ADDR=tcp://<服务端ip>: /usr/bin/rexec /usr/bin/containerd-shim $* +``` + +* /another_rootfs/usr/local/bin/remote_kill + +``` shell +#!/bin/bash +CMD_NET_ADDR=tcp://<服务端ip>: /usr/bin/rexec /usr/bin/kill $* +``` + +* /another_rootfs/usr/sbin/modprobe +``` shell +#!/bin/bash +CMD_NET_ADDR=tcp://<服务端ip>: /usr/bin/rexec /usr/sbin/modprobe $* +``` + +在chroot到dockerd和containerd运行所需的rootfs后，用如下的命令拉起dockerd和containerd + +* containerd +``` shell +#!/bin/bash +SHIM_HOST=<服务端ip> containerd --config /var/run/docker/containerd/containerd.toml --address /var/run/containerd/containerd.sock +``` + +* dockerd +``` shell +#!/bin/bash +SHIM_HOST=<服务端ip> CMD_NET_ADDR=tcp://<服务端ip>: /usr/bin/dockerd --containerd /var/run/containerd/containerd.sock +``` + +* kubelet + +在chroot环境内使用原参数拉起kubelet即可。 + +因为我们已经将/var/run/和/another_rootfs/var/run/绑定在了一起，所以可以在正常的rootfs下，通过docker来访问docker.sock接口进行容器管理。 + +至此，完成容器管理面卸载到DPU，可以通过docker相关操作进行容器创建、删除等操作，也可以通过kubectl在当前节点进行pods调度和销毁，且实际容器业务进程运行在HOST侧。 + +> ![](./public_sys-resources/icon-note.gif)**说明**： +> +> 本指导所述操作只涉及容器管理面进程卸载，不包含容器网络和数据卷volume等卸载，如有相关需求，需要通过额外的网络或存储卸载能力支持。本指导支持不带网络和存储的容器跨节点拉起。 \ No newline at end of file diff --git a/qtfs/include/comm.h b/qtfs/include/comm.h new file mode 100644 index 0000000..b275a1d --- /dev/null +++ b/qtfs/include/comm.h @@ -0,0 +1,322 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __QTFS_SERVER_COMM_H__ +#define __QTFS_SERVER_COMM_H__ + +#include + +#if (LINUX_VERSION_CODE == KERNEL_VERSION(4,19,90)) || (LINUX_VERSION_CODE == KERNEL_VERSION(4,19,36)) +#define KVER_4_19 1 +#endif + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5,6,0)) +#define BEFORE_KVER_5_6 1 +#endif + +#define QTSOCK_WL_MAX_NUM 64 +#define QTFS_WL_MAX_NUM 64 // QTFS server white list + +#define QTFS_USERP_MAXSIZE 65536 + +#define QTFS_MAX_EPEVENTS_NUM 64 + +extern struct qtinfo *qtfs_diag_info; + +#define QTFS_CLIENT_DEV "/dev/qtfs_client" +#define QTFS_SERVER_DEV "/dev/qtfs_server" + +#define QTFS_IOCTL_MAGIC 'Q' +enum { + _QTFS_IOCTL_EXEC, + _QTFS_IOCTL_THREAD_RUN, + _QTFS_IOCTL_EPFDSET, + _QTFS_IOCTL_EPOLLT, + _QTFS_IOCTL_EPOLL_THREAD_RUN, + _QTFS_IOCTL_EXIT, + + _QTFS_IOCTL_ALLINFO, + _QTFS_IOCTL_CLEARALL, + + _QTFS_IOCTL_LOG_LEVEL, + _QTFS_IOCTL_EPOLL_SUPPORT, + + _QTFS_IOCTL_WL_ADD, + _QTFS_IOCTL_WL_DEL, + _QTFS_IOCTL_WL_GET, +}; + +#define QTFS_IOCTL_THREAD_INIT _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_EXEC) +#define QTFS_IOCTL_THREAD_RUN _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_THREAD_RUN) +#define QTFS_IOCTL_EPFDSET _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_EPFDSET) +#define QTFS_IOCTL_EPOLL_THREAD_INIT _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_EPOLLT) +#define QTFS_IOCTL_EPOLL_THREAD_RUN _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_EPOLL_THREAD_RUN) +#define QTFS_IOCTL_EXIT _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_EXIT) +#define QTFS_IOCTL_ALLINFO _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_ALLINFO) +#define QTFS_IOCTL_CLEARALL _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_CLEARALL) +#define QTFS_IOCTL_LOGLEVEL _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_LOG_LEVEL) +#define QTFS_IOCTL_EPOLL_SUPPORT _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_EPOLL_SUPPORT) +#define QTFS_IOCTL_WL_ADD _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_WL_ADD) +#define QTFS_IOCTL_WL_DEL _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_WL_DEL) +#define QTFS_IOCTL_WL_GET _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_WL_GET) + +#define QTINFO_MAX_EVENT_TYPE 36 // look qtreq_type at req.h +#define QTFS_FUNCTION_LEN 64 + +#define QTFS_MAX_THREADS 16 +#define QTFS_LOGLEVEL_STRLEN 6 + +struct qtfs_server_userp_s { + size_t size; + void *userp; + void *userp2; +}; + + +enum { +#if defined(QTFS_SERVER) || defined(server) + QTFS_WHITELIST_OPEN, + QTFS_WHITELIST_WRITE, + QTFS_WHITELIST_READ, + QTFS_WHITELIST_READDIR, + QTFS_WHITELIST_MKDIR, + QTFS_WHITELIST_RMDIR, + QTFS_WHITELIST_CREATE, + QTFS_WHITELIST_UNLINK, + QTFS_WHITELIST_RENAME, + QTFS_WHITELIST_SETATTR, + QTFS_WHITELIST_SETXATTR, + QTFS_WHITELIST_MOUNT, + QTFS_WHITELIST_KILL, +#endif + QTFS_WHITELIST_UDSCONNECT, + QTFS_WHITELIST_MAX, +}; + +#define QTFS_PATH_MAX 4096 +// user-kernel struct +struct qtfs_wl_item { + unsigned int type : 10, // use to add + len : 12, // max len 4096 + index : 10; // use to query or del (index of items) + char *path; // path string +}; + +struct qtfs_thread_init_s { + unsigned int thread_nums; + struct qtfs_server_userp_s *userp; +}; + +struct qtreq_epoll_event { + unsigned int events; + unsigned long data; +}; + +struct qtfs_server_epoll_s { + int epfd; + unsigned int event_nums; + struct epoll_event *events; + struct epoll_event *kevents; +}; + +enum qtfs_errcode { + QTOK = 0, + QTERROR = 1, + QTEXIT = 2, +}; + +struct qtsock_whitelist { + unsigned int len; + char data[0]; +}; + +// qtinfo start +#if (defined(QTFS_CLIENT) || defined(client)) +enum qtinfo_cnts { + QTINF_ACTIV_CONN, + QTINF_EPOLL_ADDFDS, + QTINF_EPOLL_DELFDS, + QTINF_EPOLL_FDERR, + QTINF_SEQ_ERR, + QTINF_RESTART_SYS, + QTINF_TYPE_MISMATCH, + QTINF_NUM, +}; +#endif + +#if defined(QTFS_SERVER) || defined(server) +enum qtinfo_cnts { + QTINF_ACTIV_CONN, + QTINF_EPOLL_ADDFDS, + QTINF_EPOLL_DELFDS, + QTINF_NUM, +}; +#endif + +#if (defined(QTFS_CLIENT) || defined(client) || defined(QTFS_SERVER) || defined(server)) +// for connection state machine +typedef enum { + QTCONN_INIT, + QTCONN_CONNECTING, + QTCONN_ACTIVE, +} qtfs_conn_state_e; + +struct qtinfo_client { + unsigned long cnts[QTINF_NUM]; + unsigned long recv_err[QTINFO_MAX_EVENT_TYPE]; + unsigned long send_err[QTINFO_MAX_EVENT_TYPE]; + unsigned long i_events[QTINFO_MAX_EVENT_TYPE]; + unsigned long o_events[QTINFO_MAX_EVENT_TYPE]; + unsigned long rsp_check[QTINFO_MAX_EVENT_TYPE]; // rsp check err cnts +}; + +struct qtinfo_server { + unsigned long cnts[QTINF_NUM]; + unsigned long i_events[QTINFO_MAX_EVENT_TYPE]; + unsigned long o_events[QTINFO_MAX_EVENT_TYPE]; + unsigned long req_check[QTINFO_MAX_EVENT_TYPE]; // req check err cnts +}; + +struct qtinfo { + union { + struct qtinfo_client c; + struct qtinfo_server s; + }; + // all struct qtreq_xxx's size + unsigned int req_size[QTINFO_MAX_EVENT_TYPE]; + unsigned int rsp_size[QTINFO_MAX_EVENT_TYPE]; + int log_level; + int thread_state[QTFS_MAX_THREADS]; + char who_using[QTFS_MAX_THREADS][QTFS_FUNCTION_LEN]; + int epoll_state; + int pvar_vld; // valid param's number + int pvar_busy; // busy param's number +}; + +#define QTINFO_STATE(state) ((state == QTCONN_INIT) ? "INIT" : \ + ((state == QTCONN_CONNECTING) ? "CONNECTING" : \ + ((state == QTCONN_ACTIVE) ? "ACTIVE" : "UNKNOWN"))) +#endif + +//ko compile +#if (defined(QTFS_CLIENT) || defined(client)) +static inline void qtinfo_clear(void) +{ + int i; + for (i = QTINF_SEQ_ERR; i < QTINF_NUM; i++) + qtfs_diag_info->c.cnts[i] = 0; + memset(qtfs_diag_info->c.recv_err, 0, sizeof(qtfs_diag_info->c.recv_err)); + memset(qtfs_diag_info->c.send_err, 0, sizeof(qtfs_diag_info->c.send_err)); + memset(qtfs_diag_info->c.i_events, 0, sizeof(qtfs_diag_info->c.i_events)); + memset(qtfs_diag_info->c.o_events, 0, sizeof(qtfs_diag_info->c.o_events)); + return; +} +static inline void qtinfo_cntinc(enum qtinfo_cnts idx) +{ + if (idx >= QTINF_NUM) + return; + qtfs_diag_info->c.cnts[idx]++; + return; +} +static inline void qtinfo_cntdec(enum qtinfo_cnts idx) +{ + if (idx >= QTINF_NUM || qtfs_diag_info->c.cnts[idx] == 0) + return; + qtfs_diag_info->c.cnts[idx]--; + return; +} +static inline void qtinfo_recvinc(size_t idx) +{ + if (idx >= QTINFO_MAX_EVENT_TYPE) + return; + qtfs_diag_info->c.i_events[idx]++; + return; +} +static inline void qtinfo_sendinc(size_t idx) +{ + if (idx >= QTINFO_MAX_EVENT_TYPE) + return; + qtfs_diag_info->c.o_events[idx]++; + return; +} +static inline void qtinfo_recverrinc(size_t idx) +{ + if (idx >= QTINFO_MAX_EVENT_TYPE) + return; + qtfs_diag_info->c.recv_err[idx]++; + return; +} +static inline void qtinfo_senderrinc(size_t idx) +{ + if (idx >= QTINFO_MAX_EVENT_TYPE) + return; + qtfs_diag_info->c.send_err[idx]++; + return; +} +static inline void qtinfo_rspcheckinc(size_t idx) +{ + if (idx >= QTINFO_MAX_EVENT_TYPE) + return; + qtfs_diag_info->c.rsp_check[idx]++; + return; +} +#endif + +// ko compile +#if defined(QTFS_SERVER) || defined(server) +static inline void qtinfo_clear(void) +{ + memset(qtfs_diag_info->s.i_events, 0, sizeof(qtfs_diag_info->s.i_events)); + memset(qtfs_diag_info->s.o_events, 0, sizeof(qtfs_diag_info->s.o_events)); + return; +} +static inline void qtinfo_cntinc(enum qtinfo_cnts idx) +{ + if (idx >= QTINF_NUM) + return; + qtfs_diag_info->s.cnts[idx]++; + return; +} +static inline void qtinfo_cntdec(enum qtinfo_cnts idx) +{ + if (idx >= QTINF_NUM || qtfs_diag_info->s.cnts[idx] == 0) + return; + qtfs_diag_info->s.cnts[idx]--; + return; +} +static inline void qtinfo_recvinc(size_t idx) +{ + if (idx >= QTINFO_MAX_EVENT_TYPE) + return; + qtfs_diag_info->s.i_events[idx]++; + return; +} +static inline void qtinfo_sendinc(size_t idx) +{ + if (idx >= QTINFO_MAX_EVENT_TYPE) + return; + qtfs_diag_info->s.o_events[idx]++; + return; +} +static inline void qtinfo_reqcheckinc(size_t idx) +{ + if (idx >= QTINFO_MAX_EVENT_TYPE) + return; + qtfs_diag_info->s.req_check[idx]++; + return; +} +#endif +// QTINFO END + +#endif + diff --git a/qtfs/include/conn.h b/qtfs/include/conn.h new file mode 100644 index 0000000..ff43086 --- /dev/null +++ b/qtfs/include/conn.h @@ -0,0 +1,219 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __QTFS_CONN_H__ +#define __QTFS_CONN_H__ + +#include +#include +#include +#include +#include +#include +#include + +#include "comm.h" +#include "log.h" + +#ifdef QTFS_SERVER +extern int qtfs_server_thread_run; +extern struct qtfs_server_userp_s *qtfs_userps; +#endif +extern char qtfs_conn_type[20]; +extern char qtfs_server_ip[20]; +extern int qtfs_server_port; +extern unsigned int qtfs_server_vsock_port; +extern unsigned int qtfs_server_vsock_cid; + +extern struct qtfs_conn_var_s *qtfs_thread_var[QTFS_MAX_THREADS]; +extern struct qtfs_conn_var_s *qtfs_epoll_var; +extern char qtfs_log_level[QTFS_LOGLEVEL_STRLEN]; +extern int log_level; +extern struct qtinfo *qtfs_diag_info; +extern bool qtfs_epoll_mode; +extern struct qtfs_pvar_ops_s qtfs_conn_sock_pvar_ops; +extern struct qtfs_wl g_qtfs_wl; + +struct qtfs_conn_var_s *_qtfs_conn_get_param(const char *); +static inline struct qtfs_conn_var_s *__qtfs_conn_get_param(const char *who_using) +{ + struct qtfs_conn_var_s *p = _qtfs_conn_get_param(who_using); + if (IS_ERR_OR_NULL(p)) + return NULL; + return p; +} +#define qtfs_conn_get_param(void) __qtfs_conn_get_param(__func__) +#define qtfs_conn_get_param_errcode(void) _qtfs_conn_get_param(__func__) + +#define QTFS_CONN_SOCK_TYPE "socket" +#define QTFS_CONN_PCIE_TYPE "pcie" + +#define QTFS_EPOLL_THREADIDX (QTFS_MAX_THREADS + 4) +#define QTCONN_IS_EPOLL_CONN(pvar) (pvar->cur_threadidx == QTFS_EPOLL_THREADIDX) +#define QTFS_SERVER_MAXCONN 2 +#define QTFS_GET_PARAM_MAX_RETRY 10000 + +static inline long __must_check QTFS_PTR_ERR(__force const void *ptr) +{ + if (!ptr) + return -EINVAL; + return (long) ptr; +} + +static inline bool qtfs_support_epoll(umode_t mode) +{ + return (qtfs_epoll_mode || S_ISFIFO(mode)); +} + +#define QTFS_SOCK_RCVTIMEO 1 +#define QTFS_SOCK_SNDTIMEO 1 + +typedef enum { + QTFS_CONN_SOCKET, + QTFS_CONN_PCIE, + QTFS_CONN_INVALID, +} qtfs_conn_mode_e; + +typedef enum { + QTFS_CONN_SOCK_SERVER, + QTFS_CONN_SOCK_CLIENT, +} qtfs_conn_cs_e; + +// 使用pvar的主题类型，为了更好区分类型，socket下server有主socket下建立多个链接 +// 的需求，通过类型告知socket模块，且解藕conn层和下面的不同消息通道类型 +typedef enum { + QTFS_CONN_TYPE_QTFS, // QTFS tunnel type + QTFS_CONN_TYPE_EPOLL, // EPOLL thread type + QTFS_CONN_TYPE_FIFO, // FIFO type + QTFS_CONN_TYPE_INV, +} qtfs_conn_type_e; + +struct qtfs_pcie_var_s { + int srcid; + int dstid; +}; + +struct qtfs_sock_var_s { + struct socket *sock; + struct socket *client_sock; +#ifdef QTFS_TEST_MODE + char addr[20]; + unsigned short port; +#else + // for vsock + unsigned int vm_port; + unsigned int vm_cid; +#endif +}; + +struct qtfs_conn_ops_s { + // conn message buffer initialization and releasement. + int (*conn_var_init)(struct qtfs_conn_var_s *pvar); + void (*conn_var_fini)(struct qtfs_conn_var_s *pvar); + void (*conn_msg_clear)(struct qtfs_conn_var_s *pvar); + void *(*get_conn_msg_buf)(struct qtfs_conn_var_s *pvar, int dir); + + // connection related ops + int (*conn_init)(void *connvar, qtfs_conn_type_e type); + void (*conn_fini)(void *connvar, qtfs_conn_type_e type); + int (*conn_new_connection)(void *connvar, qtfs_conn_type_e type); + int (*conn_send)(void *connvar, void *buf, size_t len); + int (*conn_recv)(void *connvar, void *buf, size_t len, bool block); + int (*conn_send_iter)(void *connvar, struct iov_iter *iov); + int (*conn_recv_iter)(void *connvar, struct iov_iter *iov, bool block); + bool (*conn_inited)(void *connvar, qtfs_conn_type_e type); + bool (*conn_connected)(void *connvar); + void (*conn_recv_buff_drop)(void *connvar); +}; + +struct qtfs_pvar_ops_s { + // channel-specific parameter parsing function + int (*parse_param)(void); + // channel-specific global param init + int (*param_init)(void); + int (*param_fini)(void); + // init pvar with channel specific ops + int (*pvar_init)(void *connvar, struct qtfs_conn_ops_s **conn_ops, qtfs_conn_type_e type); +}; +extern struct qtfs_pvar_ops_s *g_pvar_ops; + +struct qtfs_conn_var_s { + struct list_head lst; + struct llist_node lazy_put; + int cur_threadidx; + int miss_proc; + unsigned long seq_num; + qtfs_conn_state_e state; + qtfs_conn_type_e user_type; // type of pvar's user: qtfs/epoll deamon/fifo + char who_using[QTFS_FUNCTION_LEN]; + union { + struct qtfs_sock_var_s sock_var; + struct qtfs_pcie_var_s pcie_var; + } conn_var; + struct qtfs_conn_ops_s *conn_ops; + + // use to memset buf + unsigned int recv_valid; + unsigned int send_valid; + unsigned int recv_max; + unsigned int send_max; + struct kvec vec_recv; + struct kvec vec_send; + struct iov_iter *iov_recv; // for non-copy + struct iov_iter *iov_send; // for non-copy + unsigned int magic_send; + unsigned int magic_recv; +}; + +struct qtfs_wl_cap { + unsigned int nums; + char *item[QTFS_WL_MAX_NUM]; +}; + +struct qtfs_wl { + rwlock_t rwlock; + struct qtfs_wl_cap cap[QTFS_WHITELIST_MAX]; +}; + +int qtfs_conn_init(struct qtfs_conn_var_s *pvar); +void qtfs_conn_fini(struct qtfs_conn_var_s *pvar); +int qtfs_conn_send(struct qtfs_conn_var_s *pvar); +int qtfs_conn_recv(struct qtfs_conn_var_s *pvar); +int qtfs_conn_recv_block(struct qtfs_conn_var_s *pvar); + +int qtfs_conn_var_init(struct qtfs_conn_var_s *pvar); +void qtfs_conn_var_fini(struct qtfs_conn_var_s *pvar); +void qtfs_conn_msg_clear(struct qtfs_conn_var_s *pvar); +void *qtfs_conn_msg_buf(struct qtfs_conn_var_s *pvar, int dir); + +int qtfs_conn_param_init(void); +void qtfs_conn_param_fini(void); + +void qtfs_conn_put_param(struct qtfs_conn_var_s *pvar); +struct qtfs_conn_var_s *qtfs_epoll_establish_conn(void); +void qtfs_epoll_cut_conn(struct qtfs_conn_var_s *pvar); + +#ifdef QTFS_CLIENT +struct qtfs_conn_var_s *qtfs_fifo_get_param(void); +void qtfs_fifo_put_param(struct qtfs_conn_var_s *pvar); +#endif + +int qtfs_sm_active(struct qtfs_conn_var_s *pvar); +int qtfs_sm_reconnect(struct qtfs_conn_var_s *pvar); +int qtfs_sm_exit(struct qtfs_conn_var_s *pvar); + +void qtfs_conn_list_cnt(void); + +int qtfs_uds_remote_connect_user(int fd, struct sockaddr __user *addr, int len); + +#endif diff --git a/qtfs/include/log.h b/qtfs/include/log.h new file mode 100644 index 0000000..eaf3517 --- /dev/null +++ b/qtfs/include/log.h @@ -0,0 +1,126 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __QTFS_LOG_H__ +#define __QTFS_LOG_H__ + +#include +#include "comm.h" + +enum level { + LOG_NONE, + LOG_ERROR, + LOG_WARN, + LOG_INFO, + LOG_DEBUG +}; + +extern int log_level; + +#define qtfs_crit(fmt, ...) \ + {\ + pr_crit("[%s::%s:%4d] " fmt,\ + KBUILD_MODNAME, kbasename(__FILE__), __LINE__, ##__VA_ARGS__);\ + } + +#define qtfs_err(fmt, ...) \ +( \ + { \ + if (likely(log_level >= LOG_ERROR)) { \ + pr_err("[%s::%s:%4d] " fmt, \ + KBUILD_MODNAME, kbasename(__FILE__), __LINE__, ##__VA_ARGS__); \ + } \ +} \ +) + +static inline int qtfs_log_init(char *level, int len) { + if (!strcmp(level, "WARN")) { + log_level = LOG_WARN; + } else if (!strcmp(level, "INFO")) { + log_level = LOG_INFO; + } else if (!strcmp(level, "DEBUG")) { + log_level = LOG_DEBUG; + } else if (!strcmp(level, "NONE")) { + log_level = LOG_NONE; + } else if(!strcmp(level, "ERROR")){ + log_level = LOG_ERROR; + } else { + qtfs_err("qtfs log set failed, unknown type:%s.", level); + return QTERROR; + } + return QTOK; +} + + +#define qtfs_warn(fmt, ...) \ +( \ + { \ + if (unlikely(log_level >= LOG_WARN)) { \ + pr_warn("[%s::%s:%4d] " fmt, \ + KBUILD_MODNAME, kbasename(__FILE__), __LINE__, ##__VA_ARGS__); \ + } \ +} \ +) + +#define qtfs_info(fmt, ...) \ +( \ +{ \ + if (unlikely(log_level >= LOG_INFO)) { \ + pr_info("[%s::%s:%4d] " fmt, \ + KBUILD_MODNAME, kbasename(__FILE__), __LINE__, ##__VA_ARGS__); \ + } \ +} \ +) + +#define qtfs_debug(fmt, ...) \ +( \ +{ \ + if (unlikely(log_level >= LOG_DEBUG)) { \ + pr_info("[%s::%s:%4d] " fmt, \ + KBUILD_MODNAME, kbasename(__FILE__), __LINE__, ##__VA_ARGS__); \ + } \ +} \ +) + +#define qtfs_err_ratelimited(fmt, ...) \ +( \ + { \ + if (likely(log_level >= LOG_ERROR)) { \ + pr_err_ratelimited("[%s::%s:%4d] " fmt, \ + KBUILD_MODNAME, kbasename(__FILE__), __LINE__, ##__VA_ARGS__); \ + } \ +} \ +) + +#define qtfs_info_ratelimited(fmt, ...) \ +( \ +{ \ + if (unlikely(log_level >= LOG_INFO)) { \ + pr_info_ratelimited("[%s::%s:%4d] " fmt, \ + KBUILD_MODNAME, kbasename(__FILE__), __LINE__, ##__VA_ARGS__); \ + } \ +} \ +) + +#define qtfs_warn_ratelimited(fmt, ...) \ +( \ +{ \ + if (unlikely(log_level >= LOG_WARN)) { \ + pr_warn_ratelimited("[%s::%s:%4d] " fmt, \ + KBUILD_MODNAME, kbasename(__FILE__), __LINE__, ##__VA_ARGS__); \ + } \ +} \ +) + + +#endif diff --git a/qtfs/include/qtfs_check.h b/qtfs/include/qtfs_check.h new file mode 100644 index 0000000..4794cdd --- /dev/null +++ b/qtfs/include/qtfs_check.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __QTFS_CHECK_H__ +#define __QTFS_CHECK_H__ + +#include "req.h" + +enum { + QTFS_CHECK_OK, + QTFS_CHECK_ERR, +}; + +int req_check_none(void *in); +int req_check_mount(void *in); +int req_check_open(void *in); +int req_check_close(void *in); +int req_check_readiter(void *in); +int req_check_write(void *in); +int req_check_lookup(void *in); +int req_check_readdir(void *in); +int req_check_mkdir(void *in); +int req_check_rmdir(void *in); +int req_check_getattr(void *in); +int req_check_setattr(void *in); +int req_check_icreate(void *in); +int req_check_mknod(void *in); +int req_check_unlink(void *in); +int req_check_symlink(void *in); +int req_check_link(void *in); +int req_check_getlink(void *in); +int req_check_readlink(void *in); +int req_check_rename(void *in); +int req_check_xattrlist(void *in); +int req_check_xattrget(void *in); +int req_check_xattrset(void *in); +int req_check_sysmount(void *in); +int req_check_sysumount(void *in); +int req_check_fifopoll(void *in); +int req_check_statfs(void *in); +int req_check_ioctl(void *in); +int req_check_epoll_ctl(void *in); +int req_check_llseek(void *in); +int req_check_sc_kill(void *in); +int req_check_sc_sched_getaffinity(void *in); +int req_check_sc_sched_setaffinity(void *in); + +#endif + diff --git a/qtfs/include/req.h b/qtfs/include/req.h new file mode 100644 index 0000000..70a6bf7 --- /dev/null +++ b/qtfs/include/req.h @@ -0,0 +1,562 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __QTFS_REQ_STRUCT_DEF_H__ +#define __QTFS_REQ_STRUCT_DEF_H__ + +#include +#include +#include +#include "comm.h" +#include "log.h" + +enum qtreq_type { + QTFS_REQ_NULL, + QTFS_REQ_MOUNT, + QTFS_REQ_OPEN, + QTFS_REQ_CLOSE, + QTFS_REQ_READ, + QTFS_REQ_READITER, // 5 + QTFS_REQ_WRITE, + QTFS_REQ_LOOKUP, + QTFS_REQ_READDIR, + QTFS_REQ_MKDIR, + QTFS_REQ_RMDIR, // 10 + QTFS_REQ_GETATTR, + QTFS_REQ_SETATTR, + QTFS_REQ_ICREATE, + QTFS_REQ_MKNOD, + QTFS_REQ_UNLINK, // 15 + QTFS_REQ_SYMLINK, + QTFS_REQ_LINK, + QTFS_REQ_GETLINK, + QTFS_REQ_READLINK, + QTFS_REQ_RENAME, // 20 + + QTFS_REQ_XATTRLIST, + QTFS_REQ_XATTRGET, + QTFS_REQ_XATTRSET, + + QTFS_REQ_SYSMOUNT, + QTFS_REQ_SYSUMOUNT, // 25 + QTFS_REQ_FIFOPOLL, + + QTFS_REQ_STATFS, + QTFS_REQ_IOCTL, + + QTFS_REQ_EPOLL_CTL, + + QTFS_REQ_EPOLL_EVENT, // 30 + + QTFS_REQ_LLSEEK, + + // REMOTE SYSCALL + QTFS_SC_KILL, + QTFS_SC_SCHED_GETAFFINITY, + QTFS_SC_SCHED_SETAFFINITY, + + QTFS_REQ_EXIT, // exit server thread + QTFS_REQ_INV, +}; +#define QTFS_REQ_TYPEVALID(type) (type < QTFS_REQ_INV && type >= QTFS_REQ_NULL) + + +enum qtreq_ret { + QTFS_OK, + QTFS_ERR, +}; + +enum qtfs_type { + QTFS_NORMAL, + QTFS_PROC, + QTFS_SYS, // for sysfs +}; + +struct qtfs_dirent64 { + u64 d_ino; + s64 d_off; + unsigned short d_reclen; + unsigned char d_type; + unsigned char resv[5]; + char d_name[]; +}; + +#define QTFS_SEND 0 +#define QTFS_RECV 1 + +// maximum possible length, can be increased according to the actual situation +#define NAME_MAX 255 +#define MAX_PATH_LEN PATH_MAX +#define MAX_ELSE_LEN (1024 * 128) +#define QTFS_REQ_MAX_LEN (MAX_PATH_LEN + MAX_ELSE_LEN) + +#define MAX_BUF 4096 + +// QTFS_TAIL_LEN解释： +// 私有数据结构最大长度为QTFS_REQ_MAX_LEN，超出就越界了 +// 一般有变长buf要求的，把变长buf放在末尾 +// 其长度定义为QTFS_REQ_MAX_LEN减去前面所有成员结构长度 +// 尾部变长数组长度自定义的，整体结构体长度不能超出最大长度 +#define QTFS_TAIL_LEN(head) (QTFS_REQ_MAX_LEN - sizeof(head)) + +// QTFS_SEND_SIZE解释： +// 用来发送的数据结构buf为固定大小QTFS_REQ_MAX_LEN +// 但是我们大多数时候只使用了少量的bytes +// 只需要发送有效数据，所以私有数据结构一般采取一些关键 +// 字段，加一个动态buf的组合方式，buf放在结构体末尾 +// 当传输时，只传输关键字段和动态buf的有效长度，可以用这个宏 +// 来计算所需发送的有效长度 +// 如果结构体定义不是：关键字段+字符串buf的模式，则不能用这个宏 +// 因为这个宏使用了strlen来测量末尾有效长度 +#define QTFS_SEND_SIZE(stru, tailstr) sizeof(stru) - sizeof(tailstr) + strlen(tailstr) + 1 + +struct qtreq { + unsigned int type; // operation type + unsigned int err; + unsigned long seq_num; // check code + size_t len; + char data[0]; // operation's private data +}; + +#define QTFS_MSG_LEN sizeof(struct qtreq) + QTFS_REQ_MAX_LEN +#define QTFS_MSG_HEAD_LEN sizeof(struct qtreq) + +#define QTFS_REQ_MAGIC 0xa55aa55a + +#define QTFS_FIFO_HEAD_LEN 32 // fifo只用很少的额外头，32应该足够了 +#define QTFS_FIFO_REQ_LEN (QTFS_MSG_HEAD_LEN + QTFS_FIFO_HEAD_LEN) + +struct qtreq_ioctl { + struct qtreq_ioctl_len { + unsigned int cmd; + unsigned int size; + int fd; + int argtype; // 0--use pointer arg, 1--use long arg + unsigned long arg; // for long type arg + } d; + + char path[QTFS_TAIL_LEN(struct qtreq_ioctl_len)]; +}; + +struct qtrsp_ioctl { + int ret; + int errno; + unsigned int size; + + char buf[MAX_PATH_LEN]; +}; + +struct qtreq_statfs { + char path[MAX_PATH_LEN]; // include file name +}; + +struct qtrsp_statfs { + struct kstatfs kstat; + int ret; + int errno; +}; + +struct qtreq_mount { + char path[MAX_PATH_LEN]; // include file name +}; +struct qtrsp_mount { + int ret; + int errno; +}; + +struct qtreq_open { + __u64 flags; + unsigned int mode; + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_open { + int fd; + int ret; +}; + +struct qtreq_close { + int fd; +}; + +struct qtrsp_close { + int ret; +}; + +struct qtreq_readiter { + size_t len; + long long pos; + int fd; +}; + +struct qtrsp_readiter { + struct qtrsp_readiter_len { + int ret; + ssize_t len; + int errno; + int end; + } d; + char readbuf[QTFS_TAIL_LEN(struct qtrsp_readiter_len)]; +}; + +struct qtreq_write { + struct qtreq_write_len { + int buflen; + long long pos; + int fd; + long long flags; + long long mode; + long long total_len; + } d; + // fullname and writebuf + char path_buf[QTFS_TAIL_LEN(struct qtreq_write_len)]; +}; + +struct qtrsp_write { + int ret; + ssize_t len; // 成功写入的长度 +}; + +struct qtreq_mmap { + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_mmap { + int ret; +}; + +struct qtreq_lookup { + char fullname[MAX_PATH_LEN]; +}; + +struct inode_info { + unsigned int mode; + unsigned short i_opflags; + kuid_t i_uid; + kgid_t i_gid; + unsigned int i_flags; + unsigned long i_ino; + + dev_t i_rdev; + long long i_size; + + struct timespec64 atime; + struct timespec64 mtime; + struct timespec64 ctime; + + unsigned short i_bytes; + u8 i_blkbits; + u8 i_write_hint; + blkcnt_t i_blocks; + + unsigned long i_state; + unsigned long dirtied_when; /* jiffies of first dirtying */ + unsigned long dirtied_time_when; + + __u32 i_generation; +}; + +struct qtrsp_lookup { + int ret; + int errno; + struct inode_info inode_info; +}; + +struct qtreq_readdir { + int count; + loff_t pos; + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_readdir { + struct qtrsp_readdir_len { + int ret; + int vldcnt; + int over; // 是否已经全部获取完成 + loff_t pos; + } d; + char dirent[QTFS_TAIL_LEN(struct qtrsp_readdir_len)]; +}; + +struct qtreq_mkdir { + umode_t mode; + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_mkdir { + int ret; + int errno; + struct inode_info inode_info; +}; + +struct qtreq_rmdir { + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_rmdir { + int ret; + int errno; +}; + +struct qtreq_getattr { + u32 request_mask; + unsigned int query_flags; + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_getattr { + int ret; + int errno; + struct kstat stat; +}; + +struct qtreq_setattr { + struct iattr attr; + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_setattr { + int ret; + int errno; +}; + +struct qtreq_icreate { + umode_t mode; + bool excl; + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_icreate { + int ret; + int errno; + struct inode_info inode_info; +}; + +struct qtreq_mknod { + umode_t mode; + dev_t dev; + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_mknod { + int ret; + int errno; + struct inode_info inode_info; +}; + +struct qtreq_unlink { + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_unlink { + int errno; +}; + +struct qtreq_symlink { + struct qtreq_symlink_len { + size_t newlen; + size_t oldlen; + } d; + char path[QTFS_TAIL_LEN(struct qtreq_symlink_len)]; +}; + +struct qtrsp_symlink { + int ret; + int errno; + struct inode_info inode_info; +}; + +struct qtreq_link { + struct qtreq_link_len { + size_t newlen; + size_t oldlen; + } d; + char path[QTFS_TAIL_LEN(struct qtreq_link_len)]; +}; + +struct qtrsp_link { + int ret; + int errno; + struct inode_info inode_info; +}; + +struct qtreq_getlink { + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_getlink { + int ret; + int errno; + char path[MAX_PATH_LEN]; +}; + +struct qtreq_rename { + struct qtreq_rename_len { + size_t oldlen; + size_t newlen; + size_t flags; + }d; + char path[QTFS_TAIL_LEN(struct qtreq_rename_len)]; +}; + +struct qtrsp_rename { + int ret; + int errno; +}; + +// xattr def +#define QTFS_XATTR_LEN 512 +struct qtreq_xattrlist { + size_t buffer_size; + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_xattrlist { + struct qtrsp_xattrlist_len { + int ret; + ssize_t size; + }d; + char name[QTFS_TAIL_LEN(struct qtrsp_xattrlist_len)]; +}; + +struct qtreq_xattrget { + struct qtreq_xattrget_len { + int pos; + int size; // 请求最多可以读取多少字节 + char prefix_name[QTFS_XATTR_LEN]; + }d; + char path[QTFS_TAIL_LEN(struct qtreq_xattrget_len)]; +}; + +struct qtrsp_xattrget { + struct qtrsp_xattrget_len { + int ret; + int errno; + ssize_t size; + int pos; + }d; + char buf[QTFS_TAIL_LEN(struct qtrsp_xattrget_len)]; +}; + +struct qtreq_xattrset { + struct qtreq_xattrset_len { + int flags; + size_t pathlen; + size_t namelen; + size_t valuelen; + } d; + /* buf: file path + name + value */ + char buf[QTFS_TAIL_LEN(struct qtreq_xattrset_len)]; +}; + +struct qtrsp_xattrset { + int ret; + int errno; +}; +// xattr end + +struct qtreq_sysmount { + struct qtreq_sysmount_len { + size_t dev_len; + size_t dir_len; + size_t type_len; + size_t data_len; + unsigned long flags; + } d; + char buf[QTFS_TAIL_LEN(struct qtreq_sysmount_len)]; +}; + +struct qtrsp_sysmount { + int errno; +}; + +struct qtreq_sysumount { + int flags; + char buf[MAX_PATH_LEN]; +}; + +struct qtrsp_sysumount { + int errno; +}; + +struct qtreq_poll { + int fd; +}; + +struct qtrsp_poll { + int ret; + __poll_t mask; +}; + + +struct qtreq_epollctl { + int fd; + int op; + struct qtreq_epoll_event event; +}; + +struct qtrsp_epollctl { + int ret; +}; + + +// server epoll 通知 client +#define QTFS_EPOLL_MAX_EVENTS 128 +struct qtreq_epollevt { + unsigned int event_nums; + struct qtreq_epoll_event events[QTFS_EPOLL_MAX_EVENTS]; +}; +#define QTFS_EPOLL_MSG_LEN (QTFS_MSG_HEAD_LEN + sizeof(struct qtreq_epollevt)) + +struct qtrsp_epollevt { + int ret; +}; + +struct qtreq_llseek { + loff_t off; + int whence; + int fd; +}; + +struct qtrsp_llseek { + int ret; + off_t off; +}; + +struct qtreq_sc_kill { + int pid; + int signum; +}; + +struct qtrsp_sc_kill { + long ret; +}; + +enum { + SC_GET = 0, + SC_SET, +}; +#define AFFINITY_MAX_LEN (8192 / BITS_PER_LONG) // max cpu nums 8192 +struct qtreq_sc_sched_affinity { + int type; // 0-get or 1-set + int pid; + size_t len; + unsigned long user_mask_ptr[0]; +}; + +struct qtrsp_sc_sched_affinity { + long ret; + int len; + unsigned long user_mask_ptr[0]; +}; +#endif diff --git a/qtfs/include/symbol_wrapper.h b/qtfs/include/symbol_wrapper.h new file mode 100644 index 0000000..f6ea28c --- /dev/null +++ b/qtfs/include/symbol_wrapper.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __QTFS_SYMBOL_WRAPPER_H__ +#define __QTFS_SYMBOL_WRAPPER_H__ + +#include +typedef unsigned long (*kallsyms_lookup_name_t)(const char *name); +extern kallsyms_lookup_name_t qtfs_kallsyms_lookup_name; + +struct qtfs_kallsyms { + unsigned long **sys_call_table; + + char *(*d_absolute_path)(const struct path *, char *, int); +#if (LINUX_VERSION_CODE <= KERNEL_VERSION(5, 10, 0)) + int (*__close_fd)(struct files_struct *, int); +#endif + struct task_struct *(*find_get_task_by_vpid)(pid_t nr); +}; + +extern struct qtfs_kallsyms qtfs_kern_syms; +int qtfs_kallsyms_hack_init(void); + +#ifdef QTFS_CLIENT +enum { + SYMBOL_SYSCALL_MOUNT, + SYMBOL_SYSCALL_UMOUNT, + SYMBOL_SYSCALL_EPOLL_CTL, + SYMBOL_SYSCALL_CONNECT, + SYMBOL_MAX_NUM, +}; +#endif +#ifdef QTFS_SERVER +enum { + SYMBOL_SYSCALL_CONNECT, + SYMBOL_MAX_NUM, +}; +#endif +extern unsigned long *symbols_origin[SYMBOL_MAX_NUM]; + +#ifdef __x86_64__ +// make the page writeable +static inline int make_rw(unsigned long address) +{ + unsigned int level; + pte_t *pte = lookup_address(address, &level); + pte->pte |= _PAGE_RW; + return 0; +} + +// make the page write protected +static inline int make_ro(unsigned long address) +{ + unsigned int level; + pte_t *pte = lookup_address(address, &level); + pte->pte &= ~_PAGE_RW; + return 0; +} +#endif + +#ifdef __aarch64__ +extern void (*update_mapping_prot)(phys_addr_t phys, unsigned long virt, phys_addr_t size, pgprot_t prot); +extern unsigned long start_rodata, end_rodata; +#define section_size (end_rodata - start_rodata) +#endif + +int qtfs_syscall_replace_start(void); +void qtfs_syscall_replace_stop(void); + +noinline long qtfs_syscall_umount(char __user *name, int flags); +noinline long qtfs_syscall_mount(char __user *dev_name, char __user *dir_name, + char __user *type, unsigned long flags, void __user *data); +noinline long qtfs_syscall_epoll_ctl(int epfd, int op, int fd, struct epoll_event __user *event); +noinline long qtfs_syscall_unlink(const char __user *pathname); +noinline int qtfs_syscall_readlinkat(int dfd, const char __user *path, + char __user *buf, int bufsiz); +noinline int qtfs_syscall_renameat2(int olddfd, const char __user *oldname, + int newdfd, const char __user *newname, unsigned int flags); +noinline long qtfs_syscall_mkdirat(int dfd, const char __user *pathname, umode_t mode); +noinline long qtfs_syscall_rmdir(const char __user *pathname); +noinline int qtfs_syscall_statfs(const char __user *path, struct statfs __user *buf); +noinline int qtfs_syscall_openat(int dfd, const char __user *filename, int flags, + umode_t mode); +noinline int qtfs_syscall_epoll_wait(int epfd, struct epoll_event __user *events, + int maxevents, int timout); +noinline int qtfs_syscall_linkat(int olddfd, const char __user *oldname, + int newdfd, const char __user *newname, int flags); +noinline long qtfs_syscall_mknodat(int dfd, const char __user *filename, umode_t mode, + unsigned int dev); +noinline off_t qtfs_syscall_lseek(unsigned int fd, off_t offset, unsigned int whence); +long qtfs_syscall_write(unsigned int fd, const char __user *buf, size_t count); +long qtfs_syscall_read(unsigned int fd, char __user *buf, size_t count); +long qtfs_syscall_ioctl(unsigned int fd, unsigned int cmd, unsigned long arg); +long qtfs_syscall_kill(pid_t pid, int sig); +long qtfs_syscall_sched_getaffinity(pid_t pid, unsigned int len, unsigned long __user *user_mask_ptr); +long qtfs_syscall_sched_setaffinity(pid_t pid, unsigned int len, unsigned long __user *user_mask_ptr); +long qtfs_syscall_connect(int fd, struct sockaddr __user *uservaddr, int addrlen); +#endif + diff --git a/qtfs/ipc/Makefile b/qtfs/ipc/Makefile new file mode 100644 index 0000000..383abc6 --- /dev/null +++ b/qtfs/ipc/Makefile @@ -0,0 +1,38 @@ +DEPGLIB=-lglib-2.0 -I../ -I../include/ -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include -lpthread +CFLAGS += -g -O2 +CFLAGS += -fstack-protector-strong +CFLAGS += -fPIE -pie -fPIC +CFLAGS += -D_FORTIFY_SOURCE=2 +ifdef UDS_TEST_MODE +CFLAGS += -DUDS_TEST_MODE +endif +LDFLAGS += -Wl,-z,now +LDFLAGS += -Wl,-z,noexecstack +LDFLAGS += -fPIE -pie + +all: udsproxyd libudsproxy.so + +udsproxyd: uds_event.o uds_main.o + gcc $(LDFLAGS) -o udsproxyd $^ -I../ $(DEPGLIB) + @test -z $(UDS_TEST_MODE) || echo "Important risk warning: The test mode is turned on,\ + and udsproxyd will expose the network port, which will bring security risks and is only for\ + testing! If you do not understand the risks, please don't use or compile again without\ + UDS_TEST_MODE." + +uds_event.o: + cc $(CFLAGS) -c -o uds_event.o uds_event.c $(DEPGLIB) + +uds_main.o: + cc $(CFLAGS) -c -o uds_main.o uds_main.c $(DEPGLIB) + +libudsproxy.so: + gcc $(CFLAGS) $(LDFLAGS) -o libudsproxy.so uds_connector.c --shared + +install: + yes | cp udsproxyd /usr/bin/ + yes | cp libudsproxy.so /usr/lib64/ + +clean: + @rm -rf *.o udsproxyd libudsproxy.so + +.PHONY: clean diff --git a/qtfs/ipc/uds_connector.c b/qtfs/ipc/uds_connector.c new file mode 100644 index 0000000..6aac236 --- /dev/null +++ b/qtfs/ipc/uds_connector.c @@ -0,0 +1,160 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * qtfs licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Liqiang + * Create: 2023-03-20 + * Description: + *******************************************************************************/ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "uds_module.h" + +#define LOG_OFF 1 +#if LOG_OFF +#define uds_log(info, ...) +#define uds_err(info, ...) +#else +#define uds_log(info, ...) \ + do { \ + time_t t; \ + struct tm p; \ + time(&t); \ + localtime_r(&t, &p); \ + printf("[%d/%02d/%02d %02d:%02d:%02d][LOG:%s:%3d]"info"\n", \ + p.tm_year + 1900, p.tm_mon+1, p.tm_mday, \ + p.tm_hour, p.tm_min, p.tm_sec, __func__, __LINE__, ##__VA_ARGS__); \ + } while (0); + +#define uds_err(info, ...) \ + do { \ + time_t t; \ + struct tm p; \ + time(&t); \ + localtime_r(&t, &p); \ + printf("[%d/%02d/%02d %02d:%02d:%02d][LOG:%s:%3d]"info"\n", \ + p.tm_year + 1900, p.tm_mon+1, p.tm_mday, \ + p.tm_hour, p.tm_min, p.tm_sec, __func__, __LINE__, ##__VA_ARGS__); \ + } while (0); +#endif + +static unsigned short uds_conn_get_sock_type(int sockfd) +{ + unsigned short type; + socklen_t len = 2; + int ret = getsockopt(sockfd, SOL_SOCKET, SO_TYPE, &type, &len); + if (ret < 0) { + uds_err("get sock type failed, fd:%d", sockfd); + return (unsigned short)-1; + } + uds_log("fd:%d type:%d", sockfd, type); + return type; +} + +static int uds_conn_whitelist_check(const char *path) +{ + return 1; +} + +int connect(int fd, const struct sockaddr *addrarg, socklen_t len) +{ + int sock_fd; + typeof(connect) *libcconnect = NULL; + int libcret; + const struct sockaddr_un *addr = (const struct sockaddr_un *)addrarg; + + if (libcconnect == NULL) { + libcconnect = dlsym(((void *) - 1l), "connect"); + if (libcconnect == NULL) { + uds_err("can't find connect by dlsym."); + return -1; + } + } + + libcret = (*libcconnect)(fd, addrarg, len); + if (libcret == 0 || addr->sun_family != AF_UNIX) { + // 如果本地connect成功，或者非UNIX DOMAIN SOCKET，都直接返回即可 + return libcret; + } + + if (strlen(addr->sun_path) >= (UDS_SUN_PATH_LEN - strlen(UDS_PROXY_SUFFIX))) { + uds_err("sun_path:<%s> len:%d is too large to add suffex:<%s>, so can't connect to uds proxy.", + addr->sun_path, strlen(addr->sun_path), UDS_PROXY_SUFFIX); + return libcret; + } + + uds_log("enter uds connect fd:%d sunpath:%s family:%d len:%d ", fd, addr->sun_path, addr->sun_family, len); + // 本地未连接，且是uds链接 + if (!uds_conn_whitelist_check(addr->sun_path)) { + uds_err("path:%s not in white list", addr->sun_path); + return libcret; + } + + // 尝试远端链接 + do { + int ret; + struct uds_proxy_remote_conn_req remoteconn; + struct uds_proxy_remote_conn_rsp remotersp; + struct sockaddr_un proxy = {.sun_family = AF_UNIX}; + sock_fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (sock_fd < 0) { + uds_err("create socket failed"); + return libcret; + } + + strncpy(proxy.sun_path, UDS_BUILD_CONN_ADDR, sizeof(proxy.sun_path)); + if ((*libcconnect)(sock_fd, (struct sockaddr *)&proxy, sizeof(struct sockaddr_un)) < 0) { + uds_err("can't connect to uds proxy: %s", UDS_BUILD_CONN_ADDR); + goto err_end; + } + // 这里type需要是第一个入参fd的type + remoteconn.type = uds_conn_get_sock_type(fd); + if (remoteconn.type == (unsigned short)-1) { + remoteconn.type = SOCK_STREAM; + } + memset(remoteconn.sun_path, 0, sizeof(remoteconn.sun_path)); + strncpy(remoteconn.sun_path, addr->sun_path, sizeof(remoteconn.sun_path)); + ret = send(sock_fd, &remoteconn, sizeof(remoteconn), 0); + if (ret <= 0) { + uds_err("send remote connect request failed, ret:%d errno:%d", ret, errno); + goto err_end; + } + ret = recv(sock_fd, &remotersp, sizeof(remotersp), MSG_WAITALL); + if (ret <= 0) { + uds_err("recv remote connect replay failed, ret:%d errno:%d", ret, errno); + goto err_end; + } + if (remotersp.ret == 0) { + goto err_end; + } + } while(0); + + close(sock_fd); + + struct sockaddr_un addr_proxy; + int sun_len = strlen(addr->sun_path); + memcpy(&addr_proxy, addr, sizeof(struct sockaddr_un)); + memcpy(&addr_proxy.sun_path[sun_len], UDS_PROXY_SUFFIX, strlen(UDS_PROXY_SUFFIX)); + addr_proxy.sun_path[sun_len + strlen(UDS_PROXY_SUFFIX)] = '\0'; + return (*libcconnect)(fd, (const struct sockaddr *)&addr_proxy, sizeof(struct sockaddr_un)); + +err_end: + close(sock_fd); + return libcret; +} diff --git a/qtfs/ipc/uds_event.c b/qtfs/ipc/uds_event.c new file mode 100644 index 0000000..047949b --- /dev/null +++ b/qtfs/ipc/uds_event.c @@ -0,0 +1,1096 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * qtfs licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Liqiang + * Create: 2023-03-20 + * Description: + *******************************************************************************/ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dirent.h" + +#include "uds_main.h" +#include "uds_event.h" + +int uds_event_build_step2(void *arg, int epfd, struct uds_event_global_var *p_event_var); +int uds_event_remote_build(void *arg, int epfd, struct uds_event_global_var *p_event_var); +int uds_event_build_step3(void *arg, int epfd, struct uds_event_global_var *p_event_var); +int uds_event_uds2tcp(void *arg, int epfd, struct uds_event_global_var *p_event_var); +int uds_event_tcp2uds(void *arg, int epfd, struct uds_event_global_var *p_event_var); +int uds_event_build_step4(void *arg, int epfd, struct uds_event_global_var *p_event_var); +int uds_event_tcp2pipe(void *arg, int epfd, struct uds_event_global_var *p_event_var); +int uds_event_pipe2tcp(void *arg, int epfd, struct uds_event_global_var *p_event_var); + +int uds_event_module_init(struct uds_event_global_var *p) +{ + p->msg_controllen = UDS_EVENT_BUFLEN; + p->msg_control = (char *)malloc(p->msg_controllen); + if (p->msg_control == NULL) { + uds_err("malloc msg control buf failed."); + return EVENT_ERR; + } + + p->msg_controlsendlen = UDS_EVENT_BUFLEN; + p->msg_control_send = (char *)malloc(p->msg_controlsendlen); + if (p->msg_control_send == NULL) { + goto free1; + } + + p->iov_len = UDS_EVENT_BUFLEN; + p->iov_base = (char *)malloc(p->iov_len); + if (p->iov_base == NULL) { + uds_err("malloc iov base failed."); + goto free2; + } + + p->iov_sendlen = UDS_EVENT_BUFLEN; + p->iov_base_send = (char *)malloc(p->iov_sendlen); + if (p->iov_base_send == NULL) { + goto free3; + } + + p->buflen = UDS_EVENT_BUFLEN; + p->buf = (char *)malloc(p->buflen); + if (p->buf == NULL) { + uds_err("malloc buf failed."); + goto free4; + } + return EVENT_OK; + +free4: + free(p->iov_base_send); + p->iov_base_send = NULL; + +free3: + free(p->iov_base); + p->iov_base = NULL; + +free2: + free(p->msg_control_send); + p->msg_control_send = NULL; + +free1: + free(p->msg_control); + p->msg_control = NULL; + + p->msg_controllen = 0; + p->msg_controlsendlen = 0; + p->iov_len = 0; + p->iov_sendlen = 0; + p->buflen = 0; + return EVENT_ERR; +} + +void uds_event_module_fini(struct uds_event_global_var *p) +{ + if (p->msg_control != NULL) { + free(p->msg_control); + p->msg_control = NULL; + p->msg_controllen = 0; + } + if (p->msg_control_send != NULL) { + free(p->msg_control_send); + p->msg_control_send = NULL; + p->msg_controlsendlen = 0; + } + if (p->iov_base != NULL) { + free(p->iov_base); + p->iov_base = NULL; + p->iov_len = 0; + } + if (p->iov_base_send != NULL) { + free(p->iov_base_send); + p->iov_base_send = NULL; + p->iov_sendlen = 0; + } + if (p->buf != NULL) { + free(p->buf); + p->buf = NULL; + p->buflen = 0; + } + return; +} + +int uds_event_pre_hook(struct uds_event_global_var *p_event_var) +{ + p_event_var->cur = 0; + memset(p_event_var->tofree, 0, sizeof(struct uds_event *) * UDS_EPOLL_MAX_EVENTS); + return 0; +} + +int uds_event_post_hook(struct uds_event_global_var *p_event_var) +{ + for (int i = 0; i < p_event_var->cur; i++) { + uds_log("event:%d fd:%d free by its peer", i, p_event_var->tofree[i]->fd); + uds_del_event(p_event_var->tofree[i]); + } + return 0; +} + +int uds_event_add_to_free(struct uds_event_global_var *p_event_var, struct uds_event *evt) +{ + if (evt->pipe == 1) { + uds_log("pipe event:%d no need to free peer", evt->fd); + return 0; + } + + struct uds_event *peerevt = evt->peer; + if (peerevt == NULL || p_event_var->cur >= UDS_EPOLL_MAX_EVENTS) { + uds_err("peer event add to free is NULL, my fd:%d", evt->fd); + return -1; + } + peerevt->tofree = 1; + uds_log("event fd:%d addr add to free", peerevt->fd); + p_event_var->tofree[p_event_var->cur] = peerevt; + p_event_var->cur++; + return 0; +} + +int uds_event_pre_handler(struct uds_event *evt) +{ + if (evt->tofree == 1) { + uds_log("event fd:%d marked by peer as pending deletion", evt->fd); + return EVENT_ERR; + } + return EVENT_OK; +} + +/* + * 1. accept local uds connect request + * 2. set new connection's event to build link step2 + * 3. add new connection event to epoll list + */ +int uds_event_uds_listener(void *arg, int epfd, struct uds_event_global_var *p_event_var) +{ + int connfd; + struct uds_event *evt = (struct uds_event *)arg; + if (evt == NULL) { + uds_err("param is invalid."); + return EVENT_ERR; + } + connfd = uds_sock_step_accept(evt->fd, AF_UNIX); + if (connfd <= 0) { + uds_err("conn fd error:%d", connfd); + return EVENT_ERR; + } + + uds_log("accept an new connection, fd:%d", connfd); + + if (uds_add_event(connfd, NULL, uds_event_build_step2, NULL) == NULL) { + uds_err("failed to add event,connfd:%d", connfd); + return EVENT_ERR; + } + return EVENT_OK; +} + +int uds_event_build_step2(void *arg, int epfd, struct uds_event_global_var *p_event_var) +{ + struct uds_event *evt = (struct uds_event *)arg; + if (evt == NULL) { + uds_err("param is invalid."); + return EVENT_ERR; + } + char buf[sizeof(struct uds_tcp2tcp) + sizeof(struct uds_proxy_remote_conn_req)] = {0}; + struct uds_tcp2tcp *bdmsg = (struct uds_tcp2tcp *)buf; + struct uds_proxy_remote_conn_req *msg = (struct uds_proxy_remote_conn_req *)bdmsg->data; + struct uds_proxy_remote_conn_rsp rsp; + int len; + memset(buf, 0, sizeof(buf)); + len = uds_recv_with_timeout(evt->fd, msg, sizeof(struct uds_proxy_remote_conn_req)); + if (len == 0) { + uds_err("recv err msg:%d errno:%d", len, errno); + return EVENT_DEL; + } + if (len < 0) { + uds_err("read msg error:%d errno:%d", len, errno); + goto end; + } + if (strlen(msg->sun_path) >= (UDS_SUN_PATH_LEN - strlen(UDS_PROXY_SUFFIX))) { + uds_err("sun_path:<%s> len:%d is too large to add suffex:<%s>, so can't build uds proxy server.", + msg->sun_path, strlen(msg->sun_path), UDS_PROXY_SUFFIX); + goto end; + } + if (msg->type != SOCK_STREAM && msg->type != SOCK_DGRAM) { + uds_err("uds type:%d invalid", msg->type); + goto end; + } + + struct uds_proxy_remote_conn_req *priv = (void *)malloc(sizeof(struct uds_proxy_remote_conn_req)); + if (priv == NULL) { + uds_err("malloc failed"); + goto end; + } + + struct uds_conn_arg tcp = { + .cs = UDS_SOCKET_CLIENT, + }; + int ret; + if ((ret = uds_build_tcp_connection(&tcp)) < 0) { + uds_err("step2 build tcp connection failed, return:%d", ret); + free(priv); + goto end; + } + bdmsg->msgtype = MSGCNTL_UDS; + bdmsg->msglen = sizeof(struct uds_proxy_remote_conn_req); + if (write(tcp.connfd, bdmsg, sizeof(struct uds_tcp2tcp) + sizeof(struct uds_proxy_remote_conn_req)) < 0) { + uds_err("send msg to tcp failed"); + free(priv); + close(tcp.connfd); + goto end; + } + + uds_log("step2 recv sun path:%s, add step3 event fd:%d", msg->sun_path, tcp.connfd); + memcpy(priv, msg, sizeof(struct uds_proxy_remote_conn_req)); + if (uds_add_event(tcp.connfd, evt, uds_event_build_step3, priv) == NULL) { + uds_err("failed to add event, fd:%d", tcp.connfd); + // 新事件添加失败，本事件也要删除，否则残留在中间状态 + free(priv); + close(tcp.connfd); + return EVENT_DEL; + } + return EVENT_OK; +end: + rsp.ret = 0; + write(evt->fd, &rsp, sizeof(struct uds_proxy_remote_conn_rsp)); + return EVENT_OK; +} + + +int uds_event_build_step3(void *arg, int epfd, struct uds_event_global_var *p_event_var) +{ + struct uds_event *evt = (struct uds_event *)arg; + struct uds_proxy_remote_conn_rsp msg; + int len; + memset(&msg, 0, sizeof(struct uds_proxy_remote_conn_rsp)); + len = read(evt->fd, &msg, sizeof(struct uds_proxy_remote_conn_rsp)); + if (len <= 0) { + uds_err("read error len:%d", len); + if (len == 0) + goto event_del; + return EVENT_ERR; + } + if (msg.ret == EVENT_ERR) { + uds_log("get build ack:%d, failed", msg.ret); + goto event_del; + } + + struct uds_proxy_remote_conn_req *udsmsg = (struct uds_proxy_remote_conn_req *)evt->priv; + struct uds_conn_arg uds; + + memset(&uds, 0, sizeof(struct uds_conn_arg)); + uds.cs = UDS_SOCKET_SERVER; + uds.udstype = udsmsg->type; + strncpy(uds.sun_path, udsmsg->sun_path, sizeof(uds.sun_path)); + if (strlen(uds.sun_path) + strlen(UDS_PROXY_SUFFIX) >= sizeof(uds.sun_path)) { + uds_err("invalid sunpath:%s cant add proxy suffix:%s", uds.sun_path, UDS_PROXY_SUFFIX); + goto event_del; + } + strcat(uds.sun_path, UDS_PROXY_SUFFIX); + if (uds_build_unix_connection(&uds) < 0) { + uds_err("failed to build uds server sunpath:%s", uds.sun_path); + goto event_del; + } + uds_log("remote conn build success, build uds server type:%d sunpath:%s fd:%d OK this event suspend,", + uds.udstype, uds.sun_path, uds.sockfd); + uds_event_suspend(epfd, evt); + + struct uds_event *newevt = uds_add_event(uds.sockfd, evt, uds_event_build_step4, NULL); + if (newevt == NULL) { + close(uds.sockfd); + uds_err("uds_add_event uds_event_build_step4 is failed\n"); + goto event_del; + } + evt->tmout = UDS_EVENT_WAIT_TMOUT; + newevt->tmout = UDS_EVENT_WAIT_TMOUT; + if (uds_hash_insert_dirct(event_tmout_hash, evt->fd, evt) != 0 || + uds_hash_insert_dirct(event_tmout_hash, newevt->fd, newevt) != 0) { + uds_err("add time out hash failed fd:%d %d", evt->fd, newevt->fd); + } + uds_log("Add hash key:%d-->value and key:%d-->value", evt->fd, newevt->fd); + + msg.ret = 1; + write(evt->peer->fd, &msg, sizeof(struct uds_proxy_remote_conn_rsp)); + free(evt->priv); + evt->priv = NULL; + return EVENT_OK; + +event_del: + msg.ret = 0; + write(evt->peer->fd, &msg, sizeof(struct uds_proxy_remote_conn_rsp)); + free(evt->priv); + evt->priv = NULL; + return EVENT_DEL; +} + +int uds_event_build_step4(void *arg, int epfd, struct uds_event_global_var *p_event_var) +{ + struct uds_event *evt = (struct uds_event *)arg; + int connfd = uds_sock_step_accept(evt->fd, AF_UNIX); + if (connfd < 0) { + uds_err("accept connection failed fd:%d", connfd); + return EVENT_ERR; + } + if (uds_hash_remove_dirct(event_tmout_hash, evt->fd) != 0 || + uds_hash_remove_dirct(event_tmout_hash, evt->peer->fd) != 0) { + uds_err("failed to remove time out hash fd:%d %d", evt->fd, evt->peer->fd); + } + evt->tmout = 0; + evt->peer->tmout = 0; + + struct uds_event *peerevt = (struct uds_event *)evt->peer; + peerevt->handler = uds_event_tcp2uds; + peerevt->peer = uds_add_event(connfd, peerevt, uds_event_uds2tcp, NULL); + if (peerevt->peer == NULL) { + uds_err("failed to add new event fd:%d", connfd); + uds_event_add_to_free(p_event_var, peerevt); + return EVENT_DEL; + } + + uds_log("accept new connection fd:%d, peerfd:%d frontfd:%d peerfd:%d, peerevt(fd:%d) active now", + connfd, evt->peer->fd, peerevt->fd, peerevt->peer->fd, peerevt->fd); + uds_event_insert(epfd, peerevt); + return EVENT_DEL; +} + +int uds_event_tcp_listener(void *arg, int epfd, struct uds_event_global_var *p_event_var) +{ + struct uds_event *evt = (struct uds_event *)arg; + int connfd = uds_sock_step_accept(evt->fd, AF_INET); + if (connfd <= 0) { + uds_err("tcp conn fd error:%d", connfd); + return EVENT_ERR; + } + uds_log("tcp listener event enter, new connection fd:%d.", connfd); + + if (uds_add_event(connfd, NULL, uds_event_remote_build, NULL) == NULL) { + uds_err("failed to add new event fd:%d", connfd); + return EVENT_ERR; + } + return EVENT_OK; +} + +int uds_build_connect2uds(struct uds_event *evt, struct uds_proxy_remote_conn_req *msg) +{ + struct uds_conn_arg targ; + int len = uds_recv_with_timeout(evt->fd, msg, sizeof(struct uds_proxy_remote_conn_req)); + if (len <= 0) { + uds_err("recv failed, len:%d errno:%d", len, errno); + return EVENT_ERR; + } + + targ.cs = UDS_SOCKET_CLIENT; + targ.udstype = msg->type; + memset(targ.sun_path, 0, sizeof(targ.sun_path)); + strncpy(targ.sun_path, msg->sun_path, sizeof(targ.sun_path)); + if (uds_build_unix_connection(&targ) < 0) { + uds_err("can't connect to sun_path:%s", targ.sun_path); + goto err_ack; + } + + evt->peer = uds_add_event(targ.connfd, evt, uds_event_uds2tcp, NULL); + if (evt->peer == NULL) { + uds_err("failed to add new event fd:%d", targ.connfd); + close(targ.connfd); + goto err_ack; + } + evt->handler = uds_event_tcp2uds; + + uds_log("build link req from tcp, sunpath:%s, type:%d, eventfd:%d peerfd:%d", + msg->sun_path, msg->type, targ.connfd, evt->fd); + + struct uds_proxy_remote_conn_rsp ack; + ack.ret = EVENT_OK; + + int ret = write(evt->fd, &ack, sizeof(struct uds_proxy_remote_conn_rsp)); + if (ret <= 0) { + uds_err("reply ack failed, ret:%d", ret); + return EVENT_DEL; + } + return EVENT_OK; + +err_ack: + do { + int ret; + struct uds_proxy_remote_conn_rsp ack; + ack.ret = EVENT_ERR; + ret = write(evt->fd, &ack, sizeof(struct uds_proxy_remote_conn_rsp)); + if (ret <= 0) { + uds_err("reply ack failed, ret:%d", ret); + } + } while (0); + return EVENT_DEL; +} + +int uds_build_pipe_proxy(int efd, struct uds_event *evt, struct uds_stru_scm_pipe *msg) +{ + int len = uds_recv_with_timeout(evt->fd, msg, sizeof(struct uds_stru_scm_pipe)); + if (len <= 0) { + uds_err("recv failed, len:%d errno:%d", len, errno); + return EVENT_ERR; + } + if (msg->srcfd < 0) { + uds_err("recv failed, srcid:%d", msg->srcfd); + return EVENT_ERR; + } + if (msg->dir != SCM_PIPE_READ && msg->dir != SCM_PIPE_WRITE) { + uds_err("invalid pipe dir:%d", msg->dir); + return EVENT_ERR; + } + uds_log("pipe proxy event fd:%d pipe fd:%d dir:%d", evt->fd, msg->srcfd, msg->dir); + + if (msg->dir == SCM_PIPE_READ) { + uds_add_pipe_event(msg->srcfd, evt->fd, uds_event_pipe2tcp, NULL); + // 此处必须保留evt->fd，只删除对他的监听，以及释放evt内存即可 + uds_event_suspend(efd, evt); + free(evt); + } else { + evt->pipe = 1; + evt->peerfd = msg->srcfd; + evt->handler = uds_event_tcp2pipe; + } + return EVENT_OK; +} + +int uds_event_remote_build(void *arg, int epfd, struct uds_event_global_var *p_event_var) +{ + struct uds_event *evt = (struct uds_event *)arg; + struct uds_tcp2tcp *bdmsg = (struct uds_tcp2tcp *)p_event_var->iov_base; + struct uds_proxy_remote_conn_req *msg = (struct uds_proxy_remote_conn_req *)bdmsg->data; + int len; + int ret = EVENT_OK; + memset(p_event_var->iov_base, 0, p_event_var->iov_len); + len = uds_recv_with_timeout(evt->fd, bdmsg, sizeof(struct uds_tcp2tcp)); + if (len <= 0) { + uds_err("read no msg from sock:%d, len:%d", evt->fd, len); + return EVENT_DEL; + } + + switch (bdmsg->msgtype) { + case MSGCNTL_UDS: + ret = uds_build_connect2uds(evt, msg); + break; + case MSGCNTL_PIPE: + ret = uds_build_pipe_proxy(epfd, evt, (struct uds_stru_scm_pipe *)bdmsg->data); + break; + default: + ret = EVENT_DEL; + uds_err("remote build not support msgtype %d now", bdmsg->msgtype); + break; + } + return ret; +} + +static inline mode_t uds_msg_file_mode(int fd) +{ + struct stat st; + char path[32] = {0}; + if (fstat(fd, &st) != 0) { + uds_err("get fd:%d fstat failed, errno:%d", fd, errno); + } + if (S_ISFIFO(st.st_mode)) { + uds_log("fd:%d is fifo", fd); + } + + return st.st_mode; +} + +static int uds_msg_scm_regular_file(int scmfd, int tcpfd, struct uds_event_global_var *p_event_var) +{ + int ret; + struct uds_tcp2tcp *p_msg = (struct uds_tcp2tcp *)p_event_var->buf; + struct uds_msg_scmrights *p_scmr = (struct uds_msg_scmrights *)&p_msg->data; + char *fdproc = calloc(1, UDS_PATH_MAX); + if (fdproc == NULL) { + uds_err("failed to calloc memory"); + return EVENT_ERR; + } + sprintf(fdproc, "/proc/self/fd/%d", scmfd); + ret = readlink(fdproc, p_scmr->path, UDS_PATH_MAX); + if (ret < 0) { + uds_err("readlink:%s error, ret:%d, errno:%d", fdproc, ret, errno); + free(fdproc); + close(scmfd); + return EVENT_ERR; + } + free(fdproc); + p_scmr->flags = fcntl(scmfd, F_GETFL, 0); + if (p_scmr->flags < 0 || strlen(p_scmr->path) >= sizeof(p_scmr->path)) { + uds_err("fcntl get flags failed:%d len:%d errno:%d", p_scmr->flags, strlen(p_scmr->path), errno); + close(scmfd); + return EVENT_ERR; + } + close(scmfd); + p_msg->msgtype = MSG_SCM_RIGHTS; + p_msg->msglen = sizeof(struct uds_msg_scmrights) - sizeof(p_scmr->path) + strlen(p_scmr->path) + 1; + ret = write(tcpfd, p_msg, sizeof(struct uds_tcp2tcp) + p_msg->msglen); + if (ret <= 0) { + uds_err("send scm rights msg to tcp failed, ret:%d", ret); + return EVENT_ERR; + } + uds_log("scm rights msg send to tcp, fd:%d path:%s flags:%d", scmfd, p_scmr->path, p_scmr->flags); + return EVENT_OK; +} + +static int uds_msg_scm_fifo_file(int scmfd, int tcpfd, struct uds_event_global_var *p_event_var) +{ +#define FDPATH_LEN 32 + int ret; + struct uds_tcp2tcp *p_get = (struct uds_tcp2tcp *)p_event_var->buf; + struct uds_stru_scm_pipe *p_pipe = (struct uds_stru_scm_pipe *)p_get->data; + char path[FDPATH_LEN] = {0}; + struct stat st; + p_get->msgtype = MSG_SCM_PIPE; + p_get->msglen = sizeof(struct uds_stru_scm_pipe); + + sprintf(path, "/proc/self/fd/%d", scmfd); + lstat(path, &st); + if (st.st_mode & S_IRUSR) { + p_pipe->dir = SCM_PIPE_READ; + uds_log("scm rights recv read pipe fd:%d, mode:%o", scmfd, st.st_mode); + } else if (st.st_mode & S_IWUSR) { + p_pipe->dir = SCM_PIPE_WRITE; + uds_log("scm rights recv write pipe fd:%d, mode:%o", scmfd, st.st_mode); + } else { + uds_err("scm rights recv invalid pipe, mode:%o fd:%d", st.st_mode, scmfd); + return EVENT_ERR; + } + p_pipe->srcfd = scmfd; + ret = send(tcpfd, p_get, sizeof(struct uds_tcp2tcp) + sizeof(struct uds_stru_scm_pipe), 0); + if (ret <= 0) { + uds_err("send tar get msg failed, ret:%d errno:%d", ret, errno); + return EVENT_ERR; + } + return EVENT_OK; +} + +static int uds_msg_scmrights2tcp(struct cmsghdr *cmsg, int tcpfd, struct uds_event_global_var *p_event_var) +{ + int scmfd; + mode_t mode; + + memset(p_event_var->buf, 0, p_event_var->buflen); + memcpy(&scmfd, CMSG_DATA(cmsg), sizeof(scmfd)); + if (scmfd <= 0) { + uds_err("recv invalid scm fd:%d", scmfd); + return EVENT_ERR; + } + + mode = uds_msg_file_mode(scmfd); + + switch (mode & S_IFMT) { + case S_IFREG: + uds_log("recv scmfd:%d from uds, is regular file", scmfd); + uds_msg_scm_regular_file(scmfd, tcpfd, p_event_var); + break; + case S_IFIFO: + uds_log("recv scmfd:%d from uds, is fifo", scmfd); + uds_msg_scm_fifo_file(scmfd, tcpfd, p_event_var); + break; + default: + uds_err("scm rights not support file mode:%o", mode); + break; + } + + return EVENT_OK; +} + +static int uds_msg_cmsg2tcp(struct msghdr *msg, struct uds_event *evt, struct uds_event_global_var *p_event_var) +{ + int cnt = 0; + struct cmsghdr *cmsg = CMSG_FIRSTHDR(msg); + while (cmsg != NULL) { + cnt ++; + uds_log("cmsg type:%d len:%d level:%d, tcpfd:%d", cmsg->cmsg_type, + cmsg->cmsg_len, cmsg->cmsg_level, evt->peer->fd); + switch (cmsg->cmsg_type) { + case SCM_RIGHTS: + uds_msg_scmrights2tcp(cmsg, evt->peer->fd, p_event_var); + break; + default: + uds_err("cmsg type:%d not support now", cmsg->cmsg_type); + break; + } + cmsg = CMSG_NXTHDR(msg, cmsg); + } + return cnt; +} + +static int uds_msg_scmfd_combine_msg(struct msghdr *msg, struct cmsghdr **cmsg, int *controllen, int fd) +{ + struct cmsghdr *cnxt = NULL; + if (*cmsg == NULL) { + cnxt = CMSG_FIRSTHDR(msg); + } else { + cnxt = CMSG_NXTHDR(msg, *cmsg); + } + *cmsg = cnxt; + cnxt->cmsg_level = SOL_SOCKET; + cnxt->cmsg_type = SCM_RIGHTS; + cnxt->cmsg_len = CMSG_LEN(sizeof(fd)); + memcpy(CMSG_DATA(cnxt), &fd, sizeof(fd)); + *controllen = *controllen + cnxt->cmsg_len; + return EVENT_OK; +} + +static int uds_msg_scmright_send_fd(int sock, int fd) +{ + char byte = 0; + struct iovec iov; + struct msghdr msg; + struct cmsghdr *cmsg; + char buf[CMSG_SPACE(sizeof(fd))]; + + // send at least one char + memset(&msg, 0, sizeof(msg)); + iov.iov_base = &byte; + iov.iov_len = 1; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_name = NULL; + msg.msg_namelen = 0; + + + msg.msg_control = buf; + msg.msg_controllen = sizeof(buf); + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(fd)); + // Initialize the payload + memcpy(CMSG_DATA(cmsg), &fd, sizeof(fd)); + msg.msg_controllen = cmsg->cmsg_len; + + if (sendmsg(sock, &msg, 0) != iov.iov_len) + return -1; + return 0; +} + +static int uds_msg_cmsg2uds(struct uds_tcp2tcp *msg, struct uds_event *evt) +{ + int scmfd = -1; + switch (msg->msgtype) { + case MSG_SCM_RIGHTS: { + struct uds_msg_scmrights *p_scmr = (struct uds_msg_scmrights *)&msg->data; + int ret; + scmfd = open(p_scmr->path, p_scmr->flags); + if (scmfd < 0) { + uds_err("scm rights send fd failed, scmfd:%d path:%s flags:%d", scmfd, p_scmr->path, p_scmr->flags); + return -1; + } + uds_log("scm send fd:%d path:%s flags:%d", scmfd, p_scmr->path, p_scmr->flags); + break; + } + default: + uds_err("msg type:%d not support.", msg->msgtype); + return -1; + } + return scmfd; +} + +// drop is 1, drop this msg +int uds_msg_tcp2uds_scm_pipe(struct uds_tcp2tcp *p_msg, struct uds_event *evt, int drop) +{ + int scmfd; + int fd[SCM_PIPE_NUM]; + struct uds_stru_scm_pipe *p_pipe = (struct uds_stru_scm_pipe *)p_msg->data; + int len = uds_recv_with_timeout(evt->fd, p_pipe, p_msg->msglen); + if (len <= 0) { + uds_err("recv data failed, len:%d", len); + return EVENT_DEL; + } + if (drop) { + uds_err("just drop this msg."); + return EVENT_ERR; + } + if (p_pipe->dir != SCM_PIPE_READ && p_pipe->dir != SCM_PIPE_WRITE) { + uds_err("scm pipe recv invalid pipe dir:%d, srcfd:%d", p_pipe->dir, p_pipe->srcfd); + return EVENT_ERR; + } + struct uds_conn_arg tcp = { + .cs = UDS_SOCKET_CLIENT, + }; + int ret; + if ((ret = uds_build_tcp_connection(&tcp)) < 0) { + uds_err("build tcp connection failed, return:%d", ret); + return EVENT_ERR; + } + if (pipe(fd) == -1) { + uds_err("pipe syscall error, errno:%d", errno); + close(tcp.connfd); + return EVENT_ERR; + } + if (p_pipe->dir == SCM_PIPE_READ) { + uds_log("send read pipe:%d to peer:%d", fd[SCM_PIPE_READ], evt->peer->fd); + scmfd = fd[SCM_PIPE_READ]; + // read方向，proxy读取消息并转发，此代码处是远端，所以监听tcp换发给pipe write + uds_add_pipe_event(tcp.connfd, fd[SCM_PIPE_WRITE], uds_event_tcp2pipe, NULL); + } else { + uds_log("send write pipe:%d to peer:%d", fd[SCM_PIPE_WRITE], evt->peer->fd); + scmfd = fd[SCM_PIPE_WRITE]; + // write方向，proxy读取远端代理pipe消息并转发，此处是远端，所以监听pipe read并转发给tcp + uds_add_pipe_event(fd[SCM_PIPE_READ], tcp.connfd, uds_event_pipe2tcp, NULL); + } + + p_msg->msgtype = MSGCNTL_PIPE; + p_msg->msglen = sizeof(struct uds_stru_scm_pipe); + len = write(tcp.connfd, p_msg, sizeof(struct uds_tcp2tcp) + sizeof(struct uds_stru_scm_pipe)); + if (len <= 0) { + uds_err("send pipe msg failed, len:%d", len); + return EVENT_ERR; + } + uds_log("success to build pipe fd map, dir:%d srcfd:%d tcpfd:%d readfd:%d writefd:%d", + p_pipe->dir, p_pipe->srcfd, tcp.connfd, fd[SCM_PIPE_READ], fd[SCM_PIPE_WRITE]); + + return scmfd; +} + +int uds_event_tcp2pipe(void *arg, int epfd, struct uds_event_global_var *p_event_var) +{ + struct uds_event *evt = (struct uds_event *)arg; + memset(p_event_var->iov_base, 0, p_event_var->iov_len); + int len = read(evt->fd, p_event_var->iov_base, p_event_var->iov_len); + if (len <= 0) { + uds_err("read from tcp failed, len:%d errno:%d", len, errno); + return EVENT_DEL; + } + + uds_log("tcp:%d to pipe:%d len:%d, buf:\n>>>>>>>\n%.*s\n<<<<<<<\n", evt->fd, evt->peerfd, len, len, p_event_var->iov_base); + int ret = write(evt->peerfd, p_event_var->iov_base, len); + if (ret <= 0) { + uds_err("write to pipe failed, fd:%d errno:%d", evt->peerfd, errno); + return EVENT_DEL; + } + return EVENT_OK; +} + +int uds_event_pipe2tcp(void *arg, int epfd, struct uds_event_global_var *p_event_var) +{ + struct uds_event *evt = (struct uds_event *)arg; + memset(p_event_var->iov_base, 0, p_event_var->iov_len); + int len = read(evt->fd, p_event_var->iov_base, p_event_var->iov_len); + if (len <= 0) { + uds_err("read from pipe failed, len:%d errno:%d", len, errno); + return EVENT_DEL; + } + + uds_log("pipe:%d to tcp:%d len:%d, buf:\n>>>>>>>\n%.*s\n<<<<<<<\n", evt->fd, evt->peerfd, len, len, p_event_var->iov_base); + int ret = write(evt->peerfd, p_event_var->iov_base, len); + if (ret <= 0) { + uds_err("write to tcp failed, fd:%d errno:%d", evt->peerfd, errno); + return EVENT_DEL; + } + return EVENT_OK; + +} + +int uds_msg_tcp_end_msg(int sock) +{ + struct uds_tcp2tcp end = {.msgtype = MSG_END, .msglen = 0,}; + int ret = write(sock, &end, sizeof(struct uds_tcp2tcp)); + if (ret <= 0) { + uds_err("write end msg failed, ret:%d fd:%d", ret, sock); + return EVENT_DEL; + } + return EVENT_OK; +} + +void uds_msg_init_event_buf(struct uds_event_global_var *p) +{ + memset(p->iov_base, 0, p->iov_len); + memset(p->iov_base_send, 0, p->iov_sendlen); + memset(p->msg_control, 0, p->msg_controllen); + memset(p->msg_control_send, 0, p->msg_controlsendlen); + memset(p->buf, 0, p->buflen); + return; +} + +#define TEST_BUFLEN 256 +int uds_event_uds2tcp(void *arg, int epfd, struct uds_event_global_var *p_event_var) +{ + struct uds_event *evt = (struct uds_event *)arg; + struct iovec iov; + struct msghdr msg; + struct cmsghdr *cmsg; + int cmsgcnt = 0; + int len; + + memset(&msg, 0, sizeof(msg)); + iov.iov_base = p_event_var->iov_base + sizeof(struct uds_tcp2tcp); + iov.iov_len = p_event_var->iov_len - sizeof(struct uds_tcp2tcp); + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_name = NULL; + msg.msg_namelen = 0; + + msg.msg_control = p_event_var->msg_control; + msg.msg_controllen = p_event_var->msg_controllen; + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_len = p_event_var->msg_controllen; + + len = recvmsg(evt->fd, &msg, 0); + if (len == 0) { + uds_err("recvmsg error, return:%d", len); + uds_event_add_to_free(p_event_var, evt); + return EVENT_DEL; + } + if (len < 0) { + uds_err("recvmsg error return val:%d", len); + return EVENT_ERR; + } + cmsg = CMSG_FIRSTHDR(&msg); + if (cmsg != NULL) { + uds_log("recvmsg cmsg len:%d cmsglen:%d iovlen:%d iov:%s cmsglevel:%d cmsgtype:%d", + len, cmsg->cmsg_len, iov.iov_len, iov.iov_base, cmsg->cmsg_level, cmsg->cmsg_type); + cmsgcnt = uds_msg_cmsg2tcp(&msg, evt, p_event_var); + if (len - cmsgcnt == 0) + goto endmsg; + } + + struct uds_tcp2tcp *p_msg = (struct uds_tcp2tcp *)p_event_var->iov_base; + p_msg->msgtype = MSG_NORMAL; + p_msg->msglen = len; + int ret = write(evt->peer->fd, (void *)p_msg, p_msg->msglen + sizeof(struct uds_tcp2tcp)); + if (ret <= 0) { + uds_err("write to peer:%d failed, retcode:%d len:%d", evt->peer->fd, ret, len); + return EVENT_ERR; + } + + uds_log("write iov msg to tcp success, msgtype:%d ret:%d iovlen:%d recvlen:%d udsheadlen:%d msglen:%d msg:\n>>>>>>>\n%.*s\n<<<<<<<\n", + p_msg->msgtype, ret, iov.iov_len, len, sizeof(struct uds_tcp2tcp), p_msg->msglen, p_msg->msglen, p_msg->data); +endmsg: + return uds_msg_tcp_end_msg(evt->peer->fd); +} + +static inline void uds_close_fds(int *fds, int fdnum) +{ + for (int i = 0; i < fdnum; i++) { + close(fds[i]); + } + return; +} + +int uds_event_tcp2uds(void *arg, int epfd, struct uds_event_global_var *p_event_var) +{ +#define MAX_FDS 64 + int fds[MAX_FDS] = {0}; + int fdnum = 0; + struct uds_event *evt = (struct uds_event *)arg; + struct uds_tcp2tcp *p_msg = (struct uds_tcp2tcp *)p_event_var->iov_base; + int ret; + int normal_msg_len = 0; + struct msghdr msg; + struct cmsghdr *cmsg = NULL; + struct iovec iov; + int msg_controllen = 0; + + memset(&msg, 0, sizeof(msg)); + iov.iov_base = p_event_var->iov_base_send; + iov.iov_len = 0; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_name = NULL; + msg.msg_namelen = 0; + msg.msg_control = p_event_var->msg_control_send; + msg.msg_controllen = p_event_var->msg_controlsendlen; + + while (1) { + int len = uds_recv_with_timeout(evt->fd, p_msg, sizeof(struct uds_tcp2tcp)); + if (len <= 0) { + uds_err("recv no msg maybe sock is closed, delete this tcp2uds event, len:%d.", len); + goto close_event; + } + uds_log(" type:%d len:%d len:%d", p_msg->msgtype, p_msg->msglen, len); + if (p_msg->msgtype == MSG_END) { + break; + } + if (p_msg->msglen > p_event_var->iov_len - sizeof(struct uds_tcp2tcp) || p_msg->msglen <= 0) { + uds_err("pmsg len:%d is invalid, fd:%d peerfd:%d", p_msg->msglen, evt->fd, evt->peer->fd); + continue; + } + switch(p_msg->msgtype) { + case MSG_NORMAL: + if (normal_msg_len != 0) { + uds_err("normal msg repeat recv fd:%d", evt->fd); + goto err; + } + normal_msg_len = uds_recv_with_timeout(evt->fd, p_event_var->iov_base_send, p_msg->msglen); + if (normal_msg_len <= 0) { + uds_err("recv msg error:%d fd:%d", len, evt->fd); + goto close_event; + } + iov.iov_len = normal_msg_len; + uds_log("recv normal msg len:%d str: \n>>>>>>>\n%.*s\n<<<<<<<", iov.iov_len, iov.iov_len, iov.iov_base); + break; + case MSG_SCM_RIGHTS: { + int len; + int scmfd; + struct uds_msg_scmrights *p_scm = (struct uds_msg_scmrights *) p_msg->data; + if (p_msg->msglen >= sizeof(p_scm->path)) { + uds_err("recv msg len invalid:%d", p_msg->msglen); + goto err; + } + memset(p_scm->path, 0, sizeof(p_scm->path)); + // SCM RIGHTS msg proc + len = uds_recv_with_timeout(evt->fd, p_msg->data, p_msg->msglen); + if (len <= 0) { + uds_err("recv data failed len:%d", p_msg->msglen); + return EVENT_DEL; + } + if (fdnum >= MAX_FDS) { + uds_err("Too many fds scm."); + continue; + } + scmfd = uds_msg_cmsg2uds(p_msg, evt); + if (scmfd == -1) { + goto err; + } + fds[fdnum++] = scmfd; + uds_msg_scmfd_combine_msg(&msg, &cmsg, &msg_controllen, scmfd); + break; + } + case MSG_SCM_PIPE: { + int scmfd; + scmfd = uds_msg_tcp2uds_scm_pipe(p_msg, evt, (fdnum >= MAX_FDS) ? 1 : 0); + if (scmfd == EVENT_DEL) + goto close_event; + if (scmfd < 0) + goto err; + if (fdnum >= MAX_FDS) { + uds_err("fdnum >= MAX_FDS\n"); + continue; + } + fds[fdnum++] = scmfd; + uds_msg_scmfd_combine_msg(&msg, &cmsg, &msg_controllen, scmfd); + break; + } + default: + uds_err("recv unsupport msg type:%d event fd:%d", p_msg->msgtype, evt->fd); + break; + } + } + if (msg_controllen == 0 && iov.iov_len == 0) + goto err; + msg.msg_controllen = msg_controllen; + if (iov.iov_len == 0) iov.iov_len = 1; + ret = sendmsg(evt->peer->fd, &msg, 0); + uds_log("evt:%d sendmsg len:%d, controllen:%d errno:%d", evt->fd, ret, msg_controllen, errno); + uds_close_fds(fds, fdnum); + return EVENT_OK; +err: + uds_close_fds(fds, fdnum); + return EVENT_ERR; + +close_event: + uds_close_fds(fds, fdnum); + uds_event_add_to_free(p_event_var, evt); + return EVENT_DEL; +} + +int uds_diag_is_epoll_fd(int fd) +{ + for (int i = 0; i < p_uds_var->work_thread_num; i++) { + if (fd == p_uds_var->efd[i]) + return 1; + } + return 0; +} + +int uds_diag_string(char *buf, int len) +{ + int pos = 0; + memset(buf, 0, len); + len--; + pos = snprintf(buf, len - pos, "+-----------------------------Unix Proxy Diagnostic information-------------------------+\n"); + pos += snprintf(&buf[pos], len - pos, "+ Thread nums:%d\n", p_uds_var->work_thread_num); + for (int i = 0; i < p_uds_var->work_thread_num; i++) { + pos += snprintf(&buf[pos], len - pos, "+ Thread %d events count:%d\n", i+1, p_uds_var->work_thread[i].info.events); + } + pos += snprintf(&buf[pos], len - pos, "+ Log level:%s\n", p_uds_var->logstr[p_uds_var->loglevel]); + pos += snprintf(&buf[pos], len - pos, "+---------------------------------------------------------------------------------------+\n"); + return strlen(buf); +} + +// DIAG INFO +int uds_event_diag_info(void *arg, int epfd, struct uds_event_global_var *p_event_var) +{ + int connfd; + int len; + int ret; + struct uds_event *evt = (struct uds_event *)arg; + if (evt == NULL) { + uds_err("diag info param is invalid."); + return EVENT_ERR; + } + connfd = uds_sock_step_accept(evt->fd, AF_UNIX); + if (connfd <= 0) { + uds_err("diag info conn fd error:%d", connfd); + return EVENT_ERR; + } + + uds_log("diag accept an new connection to send diag info, fd:%d", connfd); + len = uds_diag_string(p_event_var->iov_base, p_event_var->iov_len); + ret = send(connfd, p_event_var->iov_base, len, 0); + if (ret <= 0) { + uds_err("send diag info error, ret:%d len:%d", ret, len); + } + close(connfd); + return EVENT_OK; +} + +#define UDS_LOG_STR(level) (level < 0 || level >= UDS_LOG_MAX) ? p_uds_var->logstr[UDS_LOG_MAX] : p_uds_var->logstr[level] +int uds_event_debug_level(void *arg, int epfd, struct uds_event_global_var *p_event_var) +{ + int connfd; + int len; + int ret; + int cur; + struct uds_event *evt = (struct uds_event *)arg; + if (evt == NULL) { + uds_err("debug level set param is invalid."); + return EVENT_ERR; + } + connfd = uds_sock_step_accept(evt->fd, AF_UNIX); + if (connfd <= 0) { + uds_err("debug level set conn fd error:%d", connfd); + return EVENT_ERR; + } + + cur = p_uds_var->loglevel; + if (cur + 1 < UDS_LOG_MAX) { + p_uds_var->loglevel += 1; + } else { + p_uds_var->loglevel = UDS_LOG_NONE; + } + + uds_log("debug level accept a new connection, current level:%s change to:%s", UDS_LOG_STR(cur), UDS_LOG_STR(p_uds_var->loglevel)); + + len = sprintf(p_event_var->iov_base, "+---------------UDS LOG LEVEL UPDATE--------------+\n" + "+ Log level is:%s before, now change to :%s.\n" + "+-------------------------------------------------+\n", UDS_LOG_STR(cur), UDS_LOG_STR(p_uds_var->loglevel)); + + ret = send(connfd, p_event_var->iov_base, len, 0); + if (ret <= 0) { + uds_err("send debug level info error, ret:%d len:%d", ret, len); + } + close(connfd); + return EVENT_OK; +} diff --git a/qtfs/ipc/uds_event.h b/qtfs/ipc/uds_event.h new file mode 100644 index 0000000..dec840f --- /dev/null +++ b/qtfs/ipc/uds_event.h @@ -0,0 +1,81 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * qtfs licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Liqiang + * Create: 2023-03-20 + * Description: + *******************************************************************************/ + +#ifndef __QTFS_UDS_EVENT_H__ +#define __QTFS_UDS_EVENT_H__ + +#define UDS_EVENT_BUFLEN 4096 +#define UDS_PATH_MAX 1024 + +#define UDS_EVENT_WAIT_TMOUT 5 // 5s timeout + +enum EVENT_RETCODE { + EVENT_OK = 0, + EVENT_ERR = -1, + EVENT_DEL = -2, // del this event after return +}; + +enum TCP2TCP_TYPE { + MSG_NORMAL = 0xa5a5, // 消息类型从特殊数字开始，防止误识别消息 + MSG_SCM_RIGHTS, + MSG_SCM_CREDENTIALS, // unix domain 扩展消息，预留 + MSG_SCM_SECURITY, // unix domain 扩展消息，预留 + MSG_GET_TARGET, // 控制消息，用于获取对端的target fd + MSG_SCM_PIPE, // 使用SCM传递了一个pipe + MSG_END, // tcp消息的结束体 +}; + +enum TCPCNTL_TYPE { + MSGCNTL_UDS = 1, // uds代理模式 + MSGCNTL_PIPE, // pipe匿名管道代理模式 +}; + +// 因为要区分SCM_RIGHTS和普通消息，TCP到TCP需要有一个协议头 +struct uds_tcp2tcp { + int msgtype; + int msglen; // len of data + char data[0]; +}; + +struct uds_msg_scmrights { + int flags; // open flags + char path[UDS_PATH_MAX]; +}; + +enum { + SCM_PIPE_READ = 0, + SCM_PIPE_WRITE, + SCM_PIPE_NUM, +}; + +struct uds_stru_scm_pipe { + int dir; // 0: send read filedes; 1: send write filedes + // proxy通过scm rights接收到员pipe fd，后面消息回来时事件 + // 会发生变化，所以需要回消息时带上，才能建立关联 + int srcfd; +}; + +int uds_event_uds_listener(void *arg, int epfd, struct uds_event_global_var *p_event_var); +int uds_event_tcp_listener(void *arg, int epfd, struct uds_event_global_var *p_event_var); +int uds_event_diag_info(void *arg, int epfd, struct uds_event_global_var *p_event_var); +int uds_event_debug_level(void *arg, int epfd, struct uds_event_global_var *p_event_var); +int uds_event_pre_handler(struct uds_event *evt); +int uds_event_pre_hook(struct uds_event_global_var *p_event_var); +int uds_event_post_hook(struct uds_event_global_var *p_event_var); +int uds_event_module_init(struct uds_event_global_var *p_event_var); +void uds_event_module_fini(struct uds_event_global_var *p); + +#endif + diff --git a/qtfs/ipc/uds_main.c b/qtfs/ipc/uds_main.c new file mode 100644 index 0000000..d559d01 --- /dev/null +++ b/qtfs/ipc/uds_main.c @@ -0,0 +1,758 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * qtfs licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Liqiang + * Create: 2023-03-20 + * Description: + *******************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "comm.h" +#include "uds_main.h" +#include "uds_event.h" + +char *uds_log_str[UDS_LOG_MAX + 1] = {"NONE", "ERROR", "INFO", "UNKNOWN"}; +struct uds_global_var g_uds_var; +struct uds_global_var *p_uds_var = &g_uds_var; +struct uds_event_global_var *g_event_var = NULL; +GHashTable *event_tmout_hash; + +struct uds_event *uds_alloc_event() +{ + struct uds_event *p = (struct uds_event *)malloc(sizeof(struct uds_event)); + if (p == NULL) { + uds_err("malloc failed."); + return NULL; + } + memset(p, 0, sizeof(struct uds_event)); + return p; +} + +int uds_event_insert(int efd, struct uds_event *event) +{ + struct epoll_event evt; + evt.data.ptr = (void *)event; + evt.events = EPOLLIN; + if (-1 == epoll_ctl(efd, EPOLL_CTL_ADD, event->fd, &evt)) { + uds_err("epoll ctl add fd:%d event failed.", event->fd); + return -1; + } + return 0; +} + +int uds_event_suspend(int efd, struct uds_event *event) +{ + int ret = epoll_ctl(efd, EPOLL_CTL_DEL, event->fd, NULL); + if (ret != 0) { + uds_err("failed to suspend fd:%d.", event->fd); + return -1; + } + return 0; +} + +int uds_event_delete(int efd, int fd) +{ + close(fd); + return 0; +} + +int uds_recv_with_timeout(int fd, char *msg, int len) +{ +#define TMOUT_BLOCK_SIZE 1024 +#define TMOUT_UNIT_MS 20 +#define TMOUT_INTERVAL 1 +#define TMOUT_MAX_MS 1000 + int total_recv = 0; + int ret; + int tmout_ms = ((len / TMOUT_BLOCK_SIZE) + 1) * TMOUT_UNIT_MS; + if (len <= 0 || msg == NULL || fd < 0) { + uds_err("invalid param fd:%d len:%d or %s", fd, len, (msg == NULL) ? "msg is NULL" : "msg is not NULL"); + return 0; + } + if (tmout_ms > TMOUT_MAX_MS) + tmout_ms = TMOUT_MAX_MS; + do { + ret = recv(fd, &msg[total_recv], len - total_recv, 0); + if (ret < 0) { + uds_err("recv failed ret:%d errno:%d", ret, errno); + return ret; + } + total_recv += ret; + if (total_recv > len) { + uds_err("fatal error total recv:%d longger than target len:%d", total_recv, len); + return 0; + } + if (total_recv == len) { + return total_recv; + } + usleep(TMOUT_INTERVAL * 1000); + tmout_ms -= TMOUT_INTERVAL; + } while (tmout_ms > 0); + uds_err("Fatal error, the target recv len:%d and only %d length is received when it time out", len, total_recv); + return 0; +} + +#pragma GCC diagnostic ignored "-Wpointer-to-int-cast" +int uds_event_tmout_item(gpointer key, gpointer value, gpointer data) +{ + struct uds_event *evt = (struct uds_event *)value; + if (evt->tmout == 0) { + uds_err("Unexpected time out value in fd:%d", key); + goto clear; + } + evt->tmout--; + if (evt->tmout > 0) + return 0; + + uds_log("The connection was not established within 5s, the fd:%d wait was over due to a timeout.", key); +clear: + close((int)key); + free(evt); + return 1; +} +#pragma GCC diagnostic pop + +void uds_event_timeout_proc() +{ + g_hash_table_foreach_remove(event_tmout_hash, uds_event_tmout_item, NULL); +} + +void uds_main_loop(int efd, struct uds_thread_arg *arg) +{ + int n = 0; + int ret; + struct uds_event *udsevt; + struct epoll_event *evts = NULL; + struct uds_event_global_var *p_event_var = arg->p_event_var; + if (p_event_var == NULL) { + uds_err("event variable invalid."); + return; + } + + evts = calloc(UDS_EPOLL_MAX_EVENTS, sizeof(struct epoll_event)); + if (evts == NULL) { + uds_err("init calloc evts failed."); + return; + } + if (uds_event_module_init(p_event_var) == EVENT_ERR) { + uds_err("uds event module init failed, main loop not run."); + free(evts); + return; + } +#ifdef QTFS_SERVER + extern int engine_run; + while (engine_run) { +#else + while (1) { +#endif + n = epoll_wait(efd, evts, UDS_EPOLL_MAX_EVENTS, 1000); + if (n == 0) { + uds_event_timeout_proc(); + continue; + } + if (n < 0) { + uds_err("epoll wait return errcode:%d", n); + continue; + } + arg->info.events += n; + uds_event_pre_hook(p_event_var); + for (int i = 0; i < n; i++) { + udsevt = (struct uds_event *)evts[i].data.ptr; + uds_log("event fd:%d events:%d tofree:%u", udsevt->fd, evts[i].events, udsevt->tofree); + if (udsevt->handler == NULL) { + uds_err("bad event, fd:%d handler is NULL.", udsevt->fd); + continue; + } + // 预检查失败择不执行handler + if (uds_event_pre_handler(udsevt) == EVENT_ERR) { + continue; + } + ret = udsevt->handler(udsevt, efd, p_event_var); + // 此处释放当前事件，peer事件需要handler里面释放 + if (ret == EVENT_DEL) { + uds_del_event(udsevt); + } + } + uds_event_post_hook(p_event_var); + } + free(evts); + uds_log("main loop exit."); + uds_event_module_fini(p_event_var); + return; +} + +#define UDS_MAX_LISTEN_NUM 64 +int uds_build_tcp_connection(struct uds_conn_arg *arg) +{ + int family = AF_VSOCK; + if (arg->cs > UDS_SOCKET_SERVER) { + uds_err("cs type %d is error.", arg->cs); + return -1; + } +#ifdef UDS_TEST_MODE + family = AF_INET; + struct sockaddr_in sock_addr; + memset(&sock_addr, 0, sizeof(sock_addr)); + sock_addr.sin_family = AF_INET; +#else + family = AF_VSOCK; + struct sockaddr_vm sock_addr; + memset(&sock_addr, 0, sizeof(sock_addr)); + sock_addr.svm_family = AF_VSOCK; +#endif + + int sock_fd = socket(family, SOCK_STREAM, 0); + if (sock_fd < 0) { + uds_err("As %s failed, socket fd: %d, errno:%d.", + (arg->cs == UDS_SOCKET_CLIENT) ? "client" : "server", + sock_fd, errno); + return -1; + } + arg->sockfd = sock_fd; + + if (arg->cs == UDS_SOCKET_SERVER) { +#ifdef UDS_TEST_MODE + sock_addr.sin_port = htons(p_uds_var->tcp.port); + sock_addr.sin_addr.s_addr = inet_addr(p_uds_var->tcp.addr); +#else + sock_addr.svm_port = p_uds_var->vsock.port; + sock_addr.svm_cid = p_uds_var->vsock.cid; +#endif + if (bind(sock_fd, (struct sockaddr *)&sock_addr, sizeof(sock_addr)) < 0) { + uds_err("As tcp server failed, bind error, errno:%d.", + errno); + goto close_and_return; + } + if (listen(sock_fd, UDS_MAX_LISTEN_NUM) < 0) { + uds_err("As tcp server listen failed, errno:%d.", errno); + goto close_and_return; + } + } else { +#ifdef UDS_TEST_MODE + sock_addr.sin_port = htons(p_uds_var->tcp.peerport); + sock_addr.sin_addr.s_addr = inet_addr(p_uds_var->tcp.peeraddr); +#else + sock_addr.svm_port = p_uds_var->vsock.peerport; + sock_addr.svm_cid = p_uds_var->vsock.peercid; +#endif + if (connect(arg->sockfd, (struct sockaddr *)&sock_addr, sizeof(sock_addr)) < 0) { + goto close_and_return; + } + arg->connfd = sock_fd; +#ifdef UDS_TEST_MODE + uds_log("Connect to tcp server successed, ip:%s port:%u", p_uds_var->tcp.peeraddr, p_uds_var->tcp.peerport); +#else + uds_log("Connect to vsock server successed, cid:%u port:%u", p_uds_var->vsock.peercid, p_uds_var->vsock.peerport); +#endif + } + + return 0; +close_and_return: + close(sock_fd); + return -1; +} + +int uds_build_unix_connection(struct uds_conn_arg *arg) +{ + if (arg->cs > UDS_SOCKET_SERVER) { + uds_err("cs type %d is error.", arg->cs); + return -1; + } + struct sockaddr_un sock_addr = { + .sun_family = AF_UNIX, + }; + int sock_fd = socket(AF_UNIX, arg->udstype, 0); + + if (sock_fd < 0) { + uds_err("As %s failed, socket fd: %d, errno:%d.", + (arg->cs == UDS_SOCKET_CLIENT) ? "client" : "server", + sock_fd, errno); + return -1; + } + strncpy(sock_addr.sun_path, arg->sun_path, sizeof(sock_addr.sun_path)); + arg->sockfd = sock_fd; + + if (arg->cs == UDS_SOCKET_SERVER) { + unlink(sock_addr.sun_path); + if (bind(sock_fd, (struct sockaddr *)&sock_addr, sizeof(sock_addr)) < 0) { + uds_err("As uds server failed, bind error, errno:%d.", + errno); + goto close_and_return; + } + if (listen(sock_fd, UDS_MAX_LISTEN_NUM) < 0) { + uds_err("As uds server listen failed, errno:%d.", errno); + goto close_and_return; + } + } else { + if (connect(arg->sockfd, (struct sockaddr *)&sock_addr, sizeof(struct sockaddr_un)) < 0) { + goto close_and_return; + } + arg->connfd = sock_fd; + uds_log("Connect to uds server successed, sun path:%s", arg->sun_path); + } + + return 0; +close_and_return: + uds_log("close sockfd:%d and return", sock_fd); + close(sock_fd); + return -1; + +} + +int uds_sock_step_accept(int sock_fd, int family) +{ + struct sockaddr_in in_addr; + struct sockaddr_un un_addr; + socklen_t len = (family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_un); + int connfd; + if (family == AF_INET) { + connfd = accept(sock_fd, (struct sockaddr *)&in_addr, &len); + } else { + connfd = accept(sock_fd, (struct sockaddr *)&un_addr, &len); + } + if (connfd < 0) { + uds_err("Accept error:%d, errno:%d.", connfd, errno); + return connfd; + } + if (family == AF_INET) { + uds_log("Accept success, ip:%s, port:%u", + inet_ntoa(in_addr.sin_addr), + ntohs(in_addr.sin_port)); + } else { + uds_log("Accept success, sun path:%s", un_addr.sun_path); + } + return connfd; +} + +struct uds_event *uds_add_event(int fd, struct uds_event *peer, int (*handler)(void *, int, struct uds_event_global_var *), void *priv) +{ + struct uds_event *newevt = uds_alloc_event(); + int hash = fd % p_uds_var->work_thread_num; + if (newevt == NULL || p_uds_var->efd[hash] <= 0) { + uds_err("alloc event failed, efd:%d hash:%d", p_uds_var->efd[hash], hash); + if(newevt != NULL) { + free(newevt); + } + return NULL; + } + + newevt->fd = fd; + newevt->peer = peer; // 如果tcp回应，消息转回uds这个fd + newevt->handler = handler; + newevt->priv = priv; + newevt->tofree = 0; + newevt->tmout = 0; + if (uds_event_insert(p_uds_var->efd[hash], newevt) != 0) { + uds_del_event(newevt); + return NULL; + } + return newevt; +} + +struct uds_event *uds_add_pipe_event(int fd, int peerfd, int (*handler)(void *, int, struct uds_event_global_var *), void *priv) +{ + int hash = fd % p_uds_var->work_thread_num; + struct uds_event *newevt = uds_alloc_event(); + if (newevt == NULL || p_uds_var->efd[hash] <= 0) { + uds_err("alloc event failed, efd:%d", p_uds_var->efd[hash]); + return NULL; + } + + newevt->fd = fd; + newevt->peerfd = peerfd; // 如果tcp回应，消息转回uds这个fd + newevt->handler = handler; + newevt->priv = priv; + newevt->tofree = 0; + newevt->pipe = 1; + newevt->tmout = 0; + if (uds_event_insert(p_uds_var->efd[hash], newevt) != 0) { + uds_del_event(newevt); + return NULL; + } + return newevt; +} + +void uds_del_event(struct uds_event *evt) +{ + int hash = evt->fd % p_uds_var->work_thread_num; + if (evt->pipe == 1 && evt->peerfd != -1) { + // pipe是单向，peerfd没有epoll事件，所以直接关闭 + close(evt->peerfd); + evt->peerfd = -1; + } + uds_event_delete(p_uds_var->efd[hash], evt->fd); + free(evt); + return; +} + +void uds_thread_diag_init(struct uds_thread_info *info) +{ + info->events = 0; + info->fdnum = 0; +} + +void *uds_proxy_thread(void *arg) +{ + struct uds_thread_arg *parg = (struct uds_thread_arg *)arg; + // set thread name to "udsproxyd" + prctl(PR_SET_NAME, (unsigned long)"udsproxyd"); + uds_thread_diag_init(&parg->info); + uds_main_loop(parg->efd, parg); + return NULL; +} + +struct uds_event *uds_init_unix_listener(const char *addr, int (*handler)(void *, int, struct uds_event_global_var *)) +{ + struct uds_event *udsevt; + struct uds_conn_arg arg; + struct uds_conn_arg *parg = &arg; + + parg->cs = UDS_SOCKET_SERVER; + strncpy(parg->sun_path, addr, sizeof(parg->sun_path) - 1); + parg->udstype = SOCK_STREAM; + if (uds_build_unix_connection(parg) != 0) + return NULL; + udsevt = uds_add_event(parg->sockfd, NULL, handler, NULL); + if (udsevt == NULL) { + close(parg->sockfd); + uds_err("add unix listener event failed."); + return NULL; + } + return udsevt; +} + +struct uds_event *uds_init_tcp_listener() +{ + struct uds_event *tcpevt; + struct uds_conn_arg arg; + struct uds_conn_arg *parg = &arg; + parg->cs = UDS_SOCKET_SERVER; + if (uds_build_tcp_connection(parg) != 0) + return NULL; + + tcpevt = uds_add_event(parg->sockfd, NULL, uds_event_tcp_listener, NULL); + if (tcpevt == NULL) + return NULL; + return tcpevt; +} + +static inline void uds_rollback_efd(int *efd, int maxidx) +{ + for (int i = 0; i < maxidx; i++) { + close(efd[i]); + } + return; +} + +static int uds_thread_execute() +{ + pthread_t *thrd = (pthread_t *)malloc(sizeof(pthread_t) * p_uds_var->work_thread_num); + if (thrd == NULL) { + uds_err("thread info malloc failed."); + return -1; + } + + for (int i = 0; i < p_uds_var->work_thread_num; i++) { + p_uds_var->work_thread[i].p_event_var = &g_event_var[i]; + p_uds_var->work_thread[i].efd = p_uds_var->efd[i]; + (void)pthread_create(&thrd[i], NULL, uds_proxy_thread, &p_uds_var->work_thread[i]); + } + p_uds_var->loglevel = UDS_LOG_NONE; + for (int i = 0; i < p_uds_var->work_thread_num; i++) + pthread_join(thrd[i], NULL); + free(thrd); + return 0; +} + +void uds_thread_create() +{ + struct uds_conn_arg arg; + struct uds_conn_arg *parg = &arg; + struct uds_event *udsevt; + struct uds_event *tcpevt; + struct uds_event *diagevt; + struct uds_event *logevt; + int efd; + int ret; + + for (int i = 0; i < p_uds_var->work_thread_num; i++) { + efd = epoll_create1(0); + if (efd == -1) { + uds_rollback_efd(p_uds_var->efd, i); + uds_err("epoll create1 failed, i:%d.", i); + return; + } + p_uds_var->efd[i] = efd; + } + + if ((udsevt = uds_init_unix_listener(UDS_BUILD_CONN_ADDR, uds_event_uds_listener)) == NULL) + goto rollbackefd; + + if ((tcpevt = uds_init_tcp_listener()) == NULL) + goto end; + + if ((diagevt = uds_init_unix_listener(UDS_DIAG_ADDR, uds_event_diag_info)) == NULL) + goto end1; + + if ((logevt = uds_init_unix_listener(UDS_LOGLEVEL_UPD, uds_event_debug_level)) == NULL) + goto end2; + + if (chmod(UDS_BUILD_CONN_ADDR, UDS_FILE_MODE) < 0 || + chmod(UDS_DIAG_ADDR, UDS_FILE_MODE) < 0 || + chmod(UDS_LOGLEVEL_UPD, UDS_FILE_MODE) < 0) { + uds_err("set sock file mode to %o failed, errno:%d", UDS_FILE_MODE, errno); + goto end3; + } + + ret = uds_thread_execute(); + if (ret < 0) { + uds_err("uds create thread failed."); + } +end3: + uds_del_event(logevt); +end2: + uds_del_event(diagevt); +end1: + uds_del_event(tcpevt); +end: + uds_del_event(udsevt); +rollbackefd: + for (int i = 0; i < p_uds_var->work_thread_num; i++) + close(p_uds_var->efd[i]); + + return; +} + +int uds_env_prepare() +{ + DIR *dir; + if (access(UDS_BUILD_CONN_ADDR, 0) == 0) + return EVENT_OK; + + if ((dir = opendir(UDS_BUILD_CONN_DIR)) == NULL) { + if (mkdir(UDS_BUILD_CONN_DIR, 0600) < 0) { + uds_err("mkdir %s failed.", UDS_BUILD_CONN_DIR); + return EVENT_ERR; + } + } else { + closedir(dir); + } + return EVENT_OK; +} + +int uds_hash_init() +{ + event_tmout_hash = g_hash_table_new(g_direct_hash, g_direct_equal); + if (event_tmout_hash == NULL) { + uds_err("g_hash_table_new failed."); + return EVENT_ERR; + } + uds_log("init event time out hash"); + return EVENT_OK; +} + +void uds_hash_destroy() +{ + g_hash_table_destroy(event_tmout_hash); + event_tmout_hash = NULL; + return; +} + +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +int uds_hash_insert_dirct(GHashTable *table, int key, struct uds_event *value) +{ + if (g_hash_table_insert(table, (gpointer)key, value) == 0) { + uds_err("Hash table key:%d is already exist, update it.", key); + return -1; + } + uds_log("Hash insert key:%d", key); + return 0; +} + +void *uds_hash_lookup_dirct(GHashTable *table, int key) +{ + return (void *)g_hash_table_lookup(table, (gpointer)key); +} + +int uds_hash_remove_dirct(GHashTable *table, int key) +{ + if (g_hash_table_remove(table, (gpointer)key) == 0) { + uds_err("Remove key:%d from hash failed.", key); + return -1; + } + return 0; +} +#pragma GCC diagnostic pop + +static void uds_rlimit() +{ + struct rlimit lim; + + getrlimit(RLIMIT_NOFILE, &lim); + if (lim.rlim_cur >= UDS_FD_LIMIT) + return; + uds_log("uds proxy fd cur limit:%d, change to:%d", lim.rlim_cur, UDS_FD_LIMIT); + lim.rlim_cur = UDS_FD_LIMIT; + setrlimit(RLIMIT_NOFILE, &lim); + return; +} + +// port invalid range 1024~65536 +#define UDS_PORT_VALID(port) (port >= 1024 && port < 65536) +#define ARG_NUM 6 +static int uds_arg_check_and_init(int argc, char *argv[]) +{ + int myport; + int peerport; + memset(p_uds_var, 0, sizeof(struct uds_global_var)); + p_uds_var->loglevel = UDS_LOG_INFO; + p_uds_var->logstr = uds_log_str; + if (argc != ARG_NUM) { + uds_helpinfo(argv); + return -1; + } + myport = atoi(argv[3]); + peerport = atoi(argv[5]); + p_uds_var->work_thread_num = atoi(argv[1]); + if (p_uds_var->work_thread_num <= 0 || p_uds_var->work_thread_num > UDS_WORK_THREAD_MAX) { + uds_err("work thread num:%d is invalid.(must be 1~%d)", p_uds_var->work_thread_num, UDS_WORK_THREAD_MAX); + return -1; + } + if (!UDS_PORT_VALID(myport) || !UDS_PORT_VALID(peerport)) { + uds_err("local port:%d or peer port:%d invalid.(must be 1024~65535)", myport, peerport); + return -1; + } + p_uds_var->efd = (int *)malloc(sizeof(int) * p_uds_var->work_thread_num); + if (p_uds_var->efd == NULL) { + uds_err("efd malloc failed, num:%d", p_uds_var->work_thread_num); + return -1; + } + + p_uds_var->work_thread = (struct uds_thread_arg *)malloc(sizeof(struct uds_thread_arg) * p_uds_var->work_thread_num); + if (p_uds_var->work_thread == NULL) { + free(p_uds_var->efd); + uds_err("work thread var malloc failed."); + return -1; + } +#ifdef UDS_TEST_MODE + p_uds_var->tcp.port = atoi(argv[3]); + strncpy(p_uds_var->tcp.addr, argv[2], sizeof(p_uds_var->tcp.addr) - 1); + p_uds_var->tcp.peerport = atoi(argv[5]); + strncpy(p_uds_var->tcp.peeraddr, argv[4], sizeof(p_uds_var->tcp.peeraddr) - 1); + + uds_log("uds proxy param thread num:%d ip:%s port:%u peerip:%s port:%u", + p_uds_var->work_thread_num, p_uds_var->tcp.addr, p_uds_var->tcp.port, + p_uds_var->tcp.peeraddr, p_uds_var->tcp.peerport); +#else + // vsock param: + // port and peerport is checked before + p_uds_var->vsock.cid = atoi(argv[2]); + p_uds_var->vsock.port = myport; + p_uds_var->vsock.peercid = atoi(argv[4]); + p_uds_var->vsock.peerport = peerport; +#endif + g_event_var = (struct uds_event_global_var *)malloc(sizeof(struct uds_event_global_var) * p_uds_var->work_thread_num); + if (g_event_var == NULL) { + free(p_uds_var->efd); + free(p_uds_var->work_thread); + uds_err("event variable malloc failed"); + return -1; + } + return 0; +} + +static void uds_sig_pipe(int signum) +{ + return; +} + +void uds_helpinfo(char *argv[]) +{ + uds_err("Usage:"); + uds_err(" %s .", argv[0]); + uds_err("Param:"); + uds_err(" - numbers of work thread(currently only supports 1 thread)"); + uds_err(" - server ip address"); + uds_err(" - port number"); + uds_err(" - peer address"); + uds_err(" - peer port"); + return; +} + +int check_socket_lock(void) +{ + int lock_fd = open(UDS_LOCK_ADDR, O_RDONLY | O_CREAT, 0600); + if (lock_fd == -1) + return -EINVAL; + + return flock(lock_fd, LOCK_EX | LOCK_NB); +} + +#ifdef QTFS_SERVER +int uds_proxy_main(int argc, char *argv[]) +{ +#else +int main(int argc, char *argv[]) +{ + mode_t newmask = 0077; + uds_log("change uds umask from:%o to %o", umask(newmask), newmask); +#endif + if (uds_arg_check_and_init(argc, argv) < 0) { + uds_err("global var init failed."); + return -1; + } + if (uds_env_prepare() != EVENT_OK) { + uds_err("proxy prepare environment failed."); + return -1; + } + + if (check_socket_lock() < 0) { + uds_err("another proxy running"); + return -1; + } + + if (uds_hash_init() != EVENT_OK) { + uds_err("proxy hash init failed."); + return -1; + } +#ifndef QTFS_SERVER + uds_rlimit(); +#endif + signal(SIGPIPE, uds_sig_pipe); + uds_thread_create(); + uds_hash_destroy(); + + return 0; +} diff --git a/qtfs/ipc/uds_main.h b/qtfs/ipc/uds_main.h new file mode 100644 index 0000000..f69e4b3 --- /dev/null +++ b/qtfs/ipc/uds_main.h @@ -0,0 +1,160 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * qtfs licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Liqiang + * Create: 2023-03-20 + * Description: + *******************************************************************************/ + +#ifndef __QTFS_UDS_MAIN_H__ +#define __QTFS_UDS_MAIN_H__ + +#include + +#include "uds_module.h" + +#define UDS_EPOLL_MAX_EVENTS 64 +#define UDS_WORK_THREAD_MAX 1 // Temporarily only supports 1 thread +#define UDS_FD_LIMIT 65536 +#define UDS_FILE_MODE 0600 // for least privilege + +extern struct uds_global_var *p_uds_var; +extern GHashTable *event_tmout_hash; + +enum { + UDS_LOG_NONE, + UDS_LOG_ERROR, + UDS_LOG_INFO, + UDS_LOG_MAX, +}; + +#define uds_log(info, ...) \ + if (p_uds_var->loglevel >= UDS_LOG_INFO) {\ + time_t t; \ + struct tm p; \ + time(&t); \ + localtime_r(&t, &p); \ + printf("[%d/%02d/%02d %02d:%02d:%02d][LOG:%s:%3d]"info"\n", \ + p.tm_year + 1900, p.tm_mon+1, p.tm_mday, \ + p.tm_hour, p.tm_min, p.tm_sec, __func__, __LINE__, ##__VA_ARGS__); \ + } + +#define uds_err(info, ...) \ + if (p_uds_var->loglevel >= UDS_LOG_ERROR) {\ + time_t t; \ + struct tm p; \ + time(&t); \ + localtime_r(&t, &p); \ + printf("[%d/%02d/%02d %02d:%02d:%02d][ERROR:%s:%3d]"info"\n", \ + p.tm_year + 1900, p.tm_mon+1, p.tm_mday, \ + p.tm_hour, p.tm_min, p.tm_sec, __func__, __LINE__, ##__VA_ARGS__); \ + } + +enum { + UDS_THREAD_EPWAIT = 1, // epoll wait status +}; +struct uds_thread_info { + int fdnum; + + int events; + int status; +}; + +struct uds_event_global_var { + int cur; + struct uds_event *tofree[UDS_EPOLL_MAX_EVENTS]; + char *msg_control; + int msg_controllen; + char *msg_control_send; + int msg_controlsendlen; + char *iov_base; + int iov_len; + char *iov_base_send; + int iov_sendlen; + char *buf; + int buflen; +}; + +struct uds_event { + int fd; /* 本事件由这个fd触发 */ + unsigned int tofree : 1, /* 1--in to free list; 0--not */ + pipe : 1, // this is a pipe event + tmout : 4, + reserved : 26; + union { + struct uds_event *peer; /* peer event */ + int peerfd; // scm pipe 场景单向导通，只需要一个fd即可 + }; + int (*handler)(void *, int, struct uds_event_global_var *); /* event处理函数 */ + void *priv; // private data + char cpath[UDS_SUN_PATH_LEN]; + char spath[UDS_SUN_PATH_LEN]; +}; + + +struct uds_thread_arg { + int efd; + struct uds_event_global_var *p_event_var; + struct uds_thread_info info; +}; + +struct uds_global_var { + int work_thread_num; + int *efd; + struct uds_thread_arg *work_thread; + int loglevel; + char **logstr; +#ifdef UDS_TEST_MODE + struct _tcp { + char addr[20]; + unsigned short port; + char peeraddr[20]; + unsigned short peerport; + } tcp; +#else + struct _vsock { + unsigned int cid; + unsigned int port; + unsigned int peercid; + unsigned int peerport; + } vsock; +#endif +}; +enum uds_cs { + UDS_SOCKET_CLIENT = 1, + UDS_SOCKET_SERVER, +}; + +struct uds_conn_arg { + int cs; // client(1) or server(2) + + int udstype; // DGRAM or STREAM + char sun_path[UDS_SUN_PATH_LEN]; + int sockfd; + int connfd; +}; + +struct uds_event *uds_add_event(int fd, struct uds_event *peer, int (*handler)(void *, int, struct uds_event_global_var *), void *priv); +struct uds_event *uds_add_pipe_event(int fd, int peerfd, int (*handler)(void *, int, struct uds_event_global_var *), void *priv); +int uds_sock_step_accept(int sockFd, int family); +int uds_build_tcp_connection(struct uds_conn_arg *arg); +int uds_build_unix_connection(struct uds_conn_arg *arg); +void uds_del_event(struct uds_event *evt); +int uds_event_suspend(int efd, struct uds_event *event); +int uds_event_insert(int efd, struct uds_event *event); +int uds_hash_insert_dirct(GHashTable *table, int key, struct uds_event *value); +void *uds_hash_lookup_dirct(GHashTable *table, int key); +int uds_hash_remove_dirct(GHashTable *table, int key); +int uds_recv_with_timeout(int fd, char *msg, int len); + +#ifdef QTFS_SERVER +int uds_proxy_main(int argc, char *argv[]); +#endif +#endif diff --git a/qtfs/ipc/uds_module.h b/qtfs/ipc/uds_module.h new file mode 100644 index 0000000..4e199ac --- /dev/null +++ b/qtfs/ipc/uds_module.h @@ -0,0 +1,37 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * qtfs licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Liqiang + * Create: 2023-03-20 + * Description: + *******************************************************************************/ + +#ifndef __QTFS_UDS_MODULE_H__ +#define __QTFS_UDS_MODULE_H__ + +#define UDS_BUILD_CONN_ADDR "/var/run/qtfs/remote_uds.sock" +#define UDS_DIAG_ADDR "/var/run/qtfs/uds_proxy_diag.sock" +#define UDS_LOGLEVEL_UPD "/var/run/qtfs/uds_loglevel.sock" +#define UDS_LOCK_ADDR "/var/run/qtfs/uds.lock" +#define UDS_BUILD_CONN_DIR "/var/run/qtfs/" + +#define UDS_PROXY_SUFFIX ".proxy" + +#define UDS_SUN_PATH_LEN 108 // from glibc +struct uds_proxy_remote_conn_req { + unsigned short type; + unsigned short resv; + char sun_path[UDS_SUN_PATH_LEN]; +}; +struct uds_proxy_remote_conn_rsp { + int ret; +}; + +#endif diff --git a/qtfs/qtfs/CMakeLists.txt b/qtfs/qtfs/CMakeLists.txt new file mode 100644 index 0000000..ced9507 --- /dev/null +++ b/qtfs/qtfs/CMakeLists.txt @@ -0,0 +1,75 @@ +# Build Kernel modules +set(MODULE_NAME qtfs) + +## First, find the kernel build directory +execute_process( + COMMAND uname -r + OUTPUT_VARIABLE KERNEL_RELEASE + OUTPUT_STRIP_TRAILING_WHITESPACE) +set(KBUILD_DIR /lib/modules/${KERNEL_RELEASE}/build/) +find_file(KERNEL_MAKEFILE NAMES Makefile PATHS ${KBUILD_DIR} NO_DEFAULT_PATH) +if (NOT KERNEL_MAKEFILE) + message(FATAL_ERROR "There is no Makefile in ${KBUILD_DIR}!") +endif () + +## Second, gather the source files +set(COMM_SRC_FILES + ${QTFS_BASE_DIR}/qtfs_common/conn.c + ${QTFS_BASE_DIR}/qtfs_common/misc.c + ${QTFS_BASE_DIR}/qtfs_common/qtfs_check.c + ${QTFS_BASE_DIR}/qtfs_common/socket.c + ${QTFS_BASE_DIR}/qtfs_common/symbol_wrapper.c) + +set(QTFS_SRC_FILES + ${QTFS_BASE_DIR}/qtfs/fifo.c + ${QTFS_BASE_DIR}/qtfs/miss.c + ${QTFS_BASE_DIR}/qtfs/proc.c + ${QTFS_BASE_DIR}/qtfs/qtfs-mod.c + ${QTFS_BASE_DIR}/qtfs/sb.c + ${QTFS_BASE_DIR}/qtfs/syscall.c + ${QTFS_BASE_DIR}/qtfs/xattr.c) + +set(QTFS_MODULE_SRC ${COMM_SRC_FILES} ${QTFS_SRC_FILES} ${QTFS_BASE_DIR}/utils/utils.c) + +## Third, make the src files accessible to the kernel Makefile +set(MODULE_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/) +set(QTFS_KBUILD_FILE "obj-m := ${MODULE_NAME}.o") +foreach (file ${QTFS_MODULE_SRC}) + file(RELATIVE_PATH file "${CMAKE_CURRENT_SOURCE_DIR}" "${file}") + configure_file(${file} ${file} COPYONLY) + string(REPLACE ".c" ".o" file_obj "${file}") + set(QTFS_KBUILD_FILE "${QTFS_KBUILD_FILE}\n${MODULE_NAME}-y += ${file_obj}") +endforeach () + +set(QTFS_KBUILD_FILE "${QTFS_KBUILD_FILE}\nccflags-y += -I${QTFS_BASE_DIR}/include") +set(QTFS_KBUILD_FILE "${QTFS_KBUILD_FILE}\nccflags-y += -I${QTFS_BASE_DIR}/ipc") +set(QTFS_KBUILD_FILE "${QTFS_KBUILD_FILE}\nccflags-y += -I${QTFS_BASE_DIR}/qtfs") +set(QTFS_KBUILD_FILE "${QTFS_KBUILD_FILE}\nccflags-y += -I${QTFS_BASE_DIR}/utils") +set(QTFS_KBUILD_FILE "${QTFS_KBUILD_FILE}\nccflags-y += -DQTFS_CLIENT") + +if (DEFINED UDS_TEST_MODE OR DEFINED QTFS_TEST_MODE) + set(QTFS_KBUILD_FILE "${QTFS_KBUILD_FILE}\nccflags-y += -DQTFS_TEST_MODE") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DUDS_TEST_MODE") +endif () + +## Forth, generate a Kbuild file +file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/Kbuild ${QTFS_KBUILD_FILE}) + +## Fifth, add a custom target to build the module +set(MODULE_CMD ${CMAKE_MAKE_PROGRAM} -C ${KBUILD_DIR} M=${CMAKE_CURRENT_BINARY_DIR}) +add_custom_command(OUTPUT ${MODULE_NAME}.ko + COMMAND ${MODULE_CMD} modules + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${QTFS_MODULE_SRC} ${CMAKE_CURRENT_BINARY_DIR}/Kbuild + VERBATIM) +add_custom_target(${MODULE_NAME} DEPENDS ${MODULE_NAME}.ko) +add_custom_target(${MODULE_NAME}-clean COMMAND ${MODULE_CMD} clean) + +# This target helps parsing C files for IDEs like CLion +add_library(dummy-${MODULE_NAME} EXCLUDE_FROM_ALL ${QTFS_MODULE_SRC}) +target_include_directories(dummy-${MODULE_NAME} PRIVATE + ${KBUILD_DIR}/include + ${QTFS_BASE_DIR}/include + ${QTFS_BASE_DIR}/ipc + ${QTFS_BASE_DIR}/qtfs) +set_target_properties(dummy-${MODULE_NAME} PROPERTIES DEPRECATION "DO NOT BUILD THIS TARGET") \ No newline at end of file diff --git a/qtfs/qtfs/License b/qtfs/qtfs/License new file mode 100644 index 0000000..3912109 --- /dev/null +++ b/qtfs/qtfs/License @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/qtfs/qtfs/Makefile b/qtfs/qtfs/Makefile new file mode 100644 index 0000000..9b3ebe5 --- /dev/null +++ b/qtfs/qtfs/Makefile @@ -0,0 +1,27 @@ +ifdef QTFS_TEST_MODE +ccflags-y += -I$(src)/../ -I$(src)/../utils/ -I$(src)/../include/ -I$(src)/../ipc/ -I$(src) -DQTFS_CLIENT -DQTFS_TEST_MODE +else +ccflags-y += -I$(src)/../ -I$(src)/../utils/ -I$(src)/../include/ -I$(src)/../ipc/ -I$(src) -DQTFS_CLIENT +endif + +KBUILD=/lib/modules/$(shell uname -r)/build/ +COMM=../qtfs_common/ +COMMO=$(COMM)/conn.o $(COMM)/misc.o $(COMM)/symbol_wrapper.o $(COMM)/socket.o + +obj-m:=qtfs.o +qtfs-objs:=qtfs-mod.o sb.o syscall.o xattr.o proc.o miss.o fifo.o $(COMMO) ../utils/utils.o + +all: qtfs + +qtfs: + make -C $(KBUILD) M=$(PWD) modules + @test -z $(QTFS_TEST_MODE) || echo "Important risk warning: The test mode is turned on,\ + and qtfs will expose the network port, which will bring security risks and is only for\ + testing! If you do not understand the risks, please don't use or compile again without\ + QTFS_TEST_MODE." + +clean: + make -C $(KBUILD) M=$(PWD) clean + rm -rf ../*.o ../.*.o.cmd + rm -rf $(COMMO) $(COMM).*.o.cmd + rm -rf ../utils/*.o ../utils/.*.o.cmd diff --git a/qtfs/qtfs/fifo.c b/qtfs/qtfs/fifo.c new file mode 100644 index 0000000..424bae9 --- /dev/null +++ b/qtfs/qtfs/fifo.c @@ -0,0 +1,221 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "conn.h" +#include "qtfs-mod.h" +#include "req.h" +#include "log.h" + +// 对接rust，长度对齐1字节 +#pragma pack(1) +struct qtreq_fifo_open { + u64 flags; + u32 mode; + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_fifo_open { + s32 errno; +}; + +struct qtreq_fifo_read { + u64 len; +}; + +struct qtrsp_fifo_read { + s32 errno; // same as kernel errcode, 0 is ok, < 0 is errcode + u64 len; +}; + +struct qtreq_fifo_write { + u64 len; +}; + +struct qtrsp_fifo_write { + s32 errno; + u64 len; +}; + +struct qtreq_fifo_close { + // nothing +}; + +struct qtrsp_fifo_close { + // nothing +}; +#pragma pack() + +static void qtfs_fifo_put_file(struct file *file) +{ + struct qtfs_conn_var_s *pvar = file->private_data; + if (pvar == NULL) { + qtfs_err("fifo private data invalid to put"); + return; + } + qtfs_fifo_put_param(pvar); + file->private_data = NULL; + return; +} + +int qtfs_fifo_open(struct inode *inode, struct file *file) +{ + struct kvec vec_save; + unsigned int sendmax_save; + struct qtreq *req; + struct qtreq_fifo_open *fiforeq; + struct qtrsp_fifo_open *rsp; + struct qtfs_conn_var_s *pvar = NULL; + int ret; + + req = (struct qtreq *)kmalloc(sizeof(struct qtreq) + sizeof(struct qtreq_fifo_open), GFP_KERNEL); + if (req == NULL) { + qtfs_err("get fifo open memory failed."); + return -ENOMEM; + } + memset(req, 0, sizeof(struct qtreq) + sizeof(struct qtreq_fifo_open)); + + pvar = qtfs_fifo_get_param(); + if (pvar == NULL) { + qtfs_err("fifo get param failed."); + kfree(req); + return -EINVAL; + } + fiforeq = (struct qtreq_fifo_open *)req->data; + + if (qtfs_fullname(fiforeq->path, file->f_path.dentry, sizeof(fiforeq->path)) < 0) { + qtfs_err("qtfs fifo fullname failed"); + kfree(req); + qtfs_fifo_put_param(pvar); + return -EINVAL; + } + + fiforeq->flags = file->f_flags; + fiforeq->mode = file->f_mode; + qtfs_err("fifo open path:%s size req:%u size open:%u, flags:%lu mode%u", + fiforeq->path, sizeof(struct qtreq), QTFS_SEND_SIZE(struct qtreq_fifo_open, fiforeq->path), + fiforeq->flags, fiforeq->mode); + vec_save = pvar->vec_send; + sendmax_save = pvar->send_max; + pvar->vec_send.iov_base = req; + pvar->send_max = sizeof(struct qtreq) + sizeof(struct qtreq_fifo_open); + rsp = qtfs_remote_run(pvar, QTFS_REQ_OPEN, QTFS_SEND_SIZE(struct qtreq_fifo_open, fiforeq->path)); + pvar->vec_send = vec_save; + pvar->send_max = sendmax_save; + + if (IS_ERR_OR_NULL(rsp) || rsp->errno != 0) { + ret = IS_ERR_OR_NULL(rsp) ? -EFAULT : -rsp->errno; + qtfs_fifo_put_param(pvar); + qtfs_err("qtfs fifo open :%s failed mode:%o flag:%x", fiforeq->path, fiforeq->mode, fiforeq->flags); + kfree(req); + return ret; + } + kfree(req); + WARN_ON(file->private_data); + file->private_data = pvar; + return 0; +} + +ssize_t qtfs_fifo_readiter(struct kiocb *kio, struct iov_iter *iov) +{ + struct qtfs_conn_var_s *pvar = kio->ki_filp->private_data; + struct qtreq_fifo_read *req; + struct qtrsp_fifo_read *rsp; + int total = 0; + int ret; + + if (pvar == NULL || !virt_addr_valid(pvar)) { + qtfs_err("invalid fifo read req, private data is invalid"); + return -EFAULT; + } + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + req->len = iov_iter_count(iov); + pvar->vec_recv.iov_len = QTFS_MSG_HEAD_LEN + sizeof(struct qtrsp_fifo_read); + rsp = qtfs_remote_run(pvar, QTFS_REQ_READITER, sizeof(struct qtreq_fifo_read)); + if (IS_ERR_OR_NULL(rsp) || rsp->errno != 0) { + qtfs_err("remote run failed. or errno:%d", (rsp == NULL) ? -1 : rsp->errno); + return -EFAULT; + } + + while (total < rsp->len) { + ret = pvar->conn_ops->conn_recv_iter(&pvar->conn_var, iov, false); + if (ret <= 0) { + qtfs_err("recv iter from conn module ret:%d", ret); + break; + } + total += ret; + } + return total; +} + +ssize_t qtfs_fifo_writeiter(struct kiocb *kio, struct iov_iter *iov) +{ + struct qtfs_conn_var_s *pvar = kio->ki_filp->private_data; + struct qtreq_fifo_write *req; + struct qtrsp_fifo_write *rsp; + int total = 0; + + if (pvar == NULL || !virt_addr_valid(pvar)) { + qtfs_err("invalid fifo write req, private data is invalid"); + return -EFAULT; + } + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + req->len = iov_iter_count(iov); + pvar->vec_recv.iov_len = QTFS_MSG_HEAD_LEN + sizeof(struct qtrsp_fifo_write); + pvar->iov_send = iov; + rsp = qtfs_remote_run(pvar, QTFS_REQ_WRITE, sizeof(struct qtreq_fifo_write)); + if (IS_ERR_OR_NULL(rsp) || rsp->errno != 0) { + qtfs_err("fifo write remote run failed, or errno:%d", (rsp == NULL) ? -1 : rsp->errno); + return -EFAULT; + } + return rsp->len; +} + +int qtfs_fifo_release(struct inode *inode, struct file *file) +{ + struct qtfs_conn_var_s *pvar = file->private_data; + struct qtrsp_fifo_close *rsp = NULL; + if (pvar == NULL) { + qtfs_err("invalid fifo write req, private data is invalid"); + return -EFAULT; + } + pvar->vec_recv.iov_len = QTFS_MSG_HEAD_LEN; + rsp = qtfs_remote_run(pvar, QTFS_REQ_CLOSE, 0); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_err("fifo close failed"); + } + qtfs_fifo_put_file(file); + return 0; +} + +static __poll_t +qtfs_fifo_poll(struct file *filp, poll_table *wait) +{ + return 0; +} + +struct file_operations qtfsfifo_ops = { + .read_iter = qtfs_fifo_readiter, + .write_iter = qtfs_fifo_writeiter, + .open = qtfs_fifo_open, + .release = qtfs_fifo_release, + .llseek = no_llseek, + .poll = qtfs_fifo_poll, +}; \ No newline at end of file diff --git a/qtfs/qtfs/miss.c b/qtfs/qtfs/miss.c new file mode 100644 index 0000000..65c8f4e --- /dev/null +++ b/qtfs/qtfs/miss.c @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include "conn.h" +#include "qtfs-mod.h" +#include "req.h" + +#include "log.h" +#include "ops.h" + +static int miss_open(struct qtreq *miss) +{ + struct qtrsp_open *missrsp = (struct qtrsp_open *)miss->data; + struct qtreq_close *req; + struct qtrsp_close *rsp; + struct qtfs_conn_var_s *pvar = NULL; + if (missrsp == NULL) { + qtfs_err("input response is NULL."); + return QTFS_ERR; + } + if (missrsp->ret == QTFS_ERR) + return QTFS_OK; // no need to close + + pvar = qtfs_conn_get_param(); + if (pvar == NULL) { + qtfs_err("qtfs miss open pvar invalid."); + return QTFS_ERR; + } + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + req->fd = missrsp->fd; + qtfs_err("miss open proc fd:%d.", req->fd); + rsp = qtfs_remote_run(pvar, QTFS_REQ_CLOSE, sizeof(struct qtreq_close)); + qtfs_conn_put_param(pvar); + if (IS_ERR_OR_NULL(rsp)) { + return QTFS_ERR; + } + + return QTFS_OK; +} + +static struct qtmiss_ops qtfs_miss_handles[] = { + {QTFS_REQ_NULL, NULL, "null"}, + {QTFS_REQ_MOUNT, NULL, "mount"}, + {QTFS_REQ_OPEN, miss_open, "open"}, +}; + +int qtfs_missmsg_proc(struct qtfs_conn_var_s *pvar) +{ + struct qtreq *req = (struct qtreq *)pvar->vec_send.iov_base; + struct qtreq *rsp = (struct qtreq *)pvar->vec_recv.iov_base; + int ret; + qtfs_err("qtfs miss message proc req type:%u rsp type:%u.", req->type, rsp->type); + if (rsp->type > QTFS_REQ_OPEN) { + qtfs_err("qtfs miss message proc failed, type:%u invalid, req type:%u.", rsp->type, req->type); + return -EINVAL; + } + if (qtfs_miss_handles[rsp->type].misshandle == NULL) { + qtfs_err("qtfs miss message proc not support:%u, req type:%u.", rsp->type, req->type); + return -ESRCH; + } + ret = qtfs_miss_handles[rsp->type].misshandle(rsp); + if (ret != QTFS_OK) { + qtfs_err("qtfs miss message proc failed, req type:%u rsp type:%u.", req->type, rsp->type); + } + return ret; +} diff --git a/qtfs/qtfs/ops.h b/qtfs/qtfs/ops.h new file mode 100644 index 0000000..7493d3d --- /dev/null +++ b/qtfs/qtfs/ops.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __QTFS_OPS_H__ +#define __QTFS_OPS_H__ + +#include +#include + +#include "qtfs-mod.h" + +extern struct inode_operations qtfs_proc_inode_ops; +extern struct file_operations qtfs_proc_file_ops; +extern struct inode_operations qtfs_proc_sym_ops; +extern struct file_operations qtfsfifo_ops; + +enum qtfs_type qtfs_get_type(char *str); +bool is_sb_proc(struct super_block *sb); + +struct inode *qtfs_iget(struct super_block *sb, struct inode_info *ii); +const char *qtfs_getlink(struct dentry *dentry, + struct inode *inode, struct delayed_call *done); +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) +int qtfs_getattr(struct mnt_idmap *idmap, const struct path *, struct kstat *, u32, unsigned int); +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) +int qtfs_getattr(struct user_namespace *mnt_userns, const struct path *, struct kstat *, u32, unsigned int); +#else +int qtfs_getattr(const struct path *, struct kstat *, u32, unsigned int); +#endif +struct dentry * qtfs_lookup(struct inode *, struct dentry *, unsigned int); +__poll_t qtfsfifo_poll(struct file *filp, poll_table *wait); + +#endif diff --git a/qtfs/qtfs/proc.c b/qtfs/qtfs/proc.c new file mode 100644 index 0000000..70238ea --- /dev/null +++ b/qtfs/qtfs/proc.c @@ -0,0 +1,310 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include + +#include "conn.h" +#include "qtfs-mod.h" +#include "req.h" +#include "log.h" +#include "ops.h" +#include "symbol_wrapper.h" + +struct dentry *qtfs_proc_lookup(struct inode *parent_inode, struct dentry *child_dentry, unsigned int flags); +const char *qtfs_proc_getlink(struct dentry *dentry, struct inode *inode, struct delayed_call *done); + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) +int qtfs_proc_getattr(struct mnt_idmap *idmap, const struct path *path, struct kstat *stat, u32 req_mask, unsigned int flags); +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) +int qtfs_proc_getattr(struct user_namespace *mnt_userns, const struct path *path, struct kstat *stat, u32 req_mask, unsigned int flags); +#else +int qtfs_proc_getattr(const struct path *path, struct kstat *stat, u32 req_mask, unsigned int flags); +#endif + +enum qtfs_type qtfs_get_type(char *str) +{ + if (str && !strcmp(str, "proc")) + return QTFS_PROC; + return QTFS_NORMAL; +} + +bool is_sb_proc(struct super_block *sb) +{ + struct qtfs_fs_info *qfi = NULL; + + if (!sb || !sb->s_fs_info) + return false; + + qfi = sb->s_fs_info; + return qfi->type == QTFS_PROC; +} + +const char *match_str[] = { + // k8s isula and docker + "bash", + "isula", + "isulad", + "isulad-real", + "kubelet", + "kubelet-real", + "dockerd", + "dockerd-real", + "containerd", + "containerd-real", + // Virtualization scene + "libvirtd", + "virsh", + "rpc-worker", + "rexec", +}; + +int is_local_process(const char *path) +{ + int pid = -1; + char cmdline[TASK_COMM_LEN]; + char *pos = NULL; + struct task_struct *t = NULL; + int i = 0; + + sscanf(path, "/proc/%d", &pid); + if (pid <= 0) + return -1; + + t = qtfs_kern_syms.find_get_task_by_vpid((pid_t)pid); + if (!t) { + qtfs_info("[is_local_process] Failed to get task_struct from pid(%d)", pid); + return -1; + } + get_task_comm(cmdline, t); + put_task_struct(t); + + pos = strrchr(cmdline, '/'); + if (!pos) { + pos = cmdline; + } else { + pos++; + } + + for (i = 0; i < sizeof(match_str)/sizeof(char *); i++) { + if (!strncmp(pos, match_str[i], NAME_MAX)) { + qtfs_debug("[is_local_process] cmdline: %s is local process %d\n", cmdline, pid); + return pid; + } + } + + qtfs_debug("[is_local_process] cmdline: %s is not local process", cmdline); + return -1; +} + +struct inode_operations qtfs_proc_inode_ops = { + .lookup = qtfs_proc_lookup, + .getattr = qtfs_proc_getattr, +}; + +struct inode_operations qtfs_proc_sym_ops = { + .get_link = qtfs_proc_getlink, + .getattr = qtfs_proc_getattr, +}; + +struct dentry *qtfs_proc_lookup(struct inode *parent_inode, struct dentry *child_dentry, unsigned int flags) +{ + char *cpath = NULL, *tmp = NULL; + struct path spath; + struct dentry *d = NULL; + struct inode_info ii; + struct inode *inode = NULL; + int ret = 0; + int pid = -1; + + cpath = kmalloc(MAX_PATH_LEN, GFP_KERNEL); + tmp = kmalloc(MAX_PATH_LEN, GFP_KERNEL); + if (!tmp || !cpath) { + qtfs_err("%s: failed to alloc memory", __func__); + goto remote; + } + memset(cpath, 0, MAX_PATH_LEN); + memset(tmp, 0, MAX_PATH_LEN); + + if (qtfs_fullname(cpath, child_dentry, MAX_PATH_LEN) < 0) { + qtfs_err("%s: failed to get fullname", __func__); + goto remote; + } + + pid = is_local_process(cpath); + if (pid > 0) { + sscanf(cpath, "/proc/%s", tmp); + memset(cpath, 0, MAX_PATH_LEN); + sprintf(cpath, "/local_proc/%s", tmp); + qtfs_debug("[%s]: get path from local: %s\n", __func__, cpath); + ret = kern_path(cpath, 0, &spath); + if(ret) { + qtfs_err("[%s]: kern_path(%s) failed: %d\n", __func__, cpath, ret); + goto remote; + } + + ii.mode = spath.dentry->d_inode->i_mode; + ii.mode = (ii.mode & ~(S_IFMT)) | S_IFLNK; + ii.i_size = spath.dentry->d_inode->i_size; + ii.i_ino = spath.dentry->d_inode->i_ino; + ii.atime = spath.dentry->d_inode->i_atime; + ii.mtime = spath.dentry->d_inode->i_mtime; + ii.ctime = spath.dentry->d_inode->i_ctime; + path_put(&spath); + + kfree(tmp); + tmp = NULL; + inode = qtfs_iget(parent_inode->i_sb, &ii); + if (inode == NULL) { + qtfs_err("%s: failed to get inode for %s", __func__, cpath); + kfree(cpath); + return ERR_PTR(-ENOMEM); + } + d = d_splice_alias(inode, child_dentry); + kfree(cpath); + return d; + } + +remote: + if (cpath) + kfree(cpath); + if (tmp) + kfree(tmp); + return qtfs_lookup(parent_inode, child_dentry, flags); +} + +const char *qtfs_proc_getlink(struct dentry *dentry, + struct inode *inode, struct delayed_call *done) +{ + char *link = NULL, *path = NULL, *tmp = NULL; + int pid = -1; + + link = kmalloc(MAX_PATH_LEN, GFP_KERNEL); + path = kmalloc(MAX_PATH_LEN, GFP_KERNEL); + tmp = kmalloc(MAX_PATH_LEN, GFP_KERNEL); + if (!link || !tmp || !path) { + qtfs_err("[%s]: failed to alloc memory", __func__); + goto link_remote; + } + memset(link, 0, MAX_PATH_LEN); + memset(path, 0, MAX_PATH_LEN); + memset(tmp, 0, MAX_PATH_LEN); + + if (qtfs_fullname(path, dentry, MAX_PATH_LEN) < 0) { + qtfs_info("[%s]: get path failed", __func__); + goto link_remote; + } + + if (!strncmp(path, "/proc/self", 11)) { + sprintf(link, "/local_proc/%d", (int)current->pid); + qtfs_info("[%s] success: %s getlink: %s", __func__, path, link); + goto link_local; + } + + if (!strcmp(path, "/proc/mounts")) { + sprintf(link, "/proc/1/mounts"); + qtfs_info("[%s] success: %s getlink /proc/1/mounts", __func__, path); + goto link_local; + } + + pid = is_local_process(path); + if (pid > 0) { + sscanf(path, "/proc/%s", tmp); + sprintf(link, "/local_proc/%s", tmp); + qtfs_info("[%s] success: %s getlink: %s", __func__, path, link); + goto link_local; + } + +link_remote: + if (link) + kfree(link); + if (tmp) + kfree(tmp); + if (path) + kfree(path); + return qtfs_getlink(dentry, inode, done); +link_local: + kfree(tmp); + kfree(path); + set_delayed_call(done, kfree_link, link); + return link; +} + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) +int qtfs_proc_getattr(struct mnt_idmap *idmap, const struct path *path, struct kstat *stat, u32 req_mask, unsigned int flags) +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) +int qtfs_proc_getattr(struct user_namespace *mnt_userns, const struct path *path, struct kstat *stat, u32 req_mask, unsigned int flags) +#else +int qtfs_proc_getattr(const struct path *path, struct kstat *stat, u32 req_mask, unsigned int flags) +#endif +{ + char *cpath = NULL, *tmp = NULL, *local_path = NULL; + struct path spath; + int ret = 0; + int pid = -1; + + cpath = kmalloc(MAX_PATH_LEN, GFP_KERNEL); + tmp = kmalloc(MAX_PATH_LEN, GFP_KERNEL); + local_path = kmalloc(MAX_PATH_LEN, GFP_KERNEL); + if (!cpath || !tmp || !local_path) { + qtfs_err("[%s]: failed to alloc memory", __func__); + goto remote; + } + memset(cpath, 0, MAX_PATH_LEN); + memset(tmp, 0, MAX_PATH_LEN); + memset(local_path, 0, MAX_PATH_LEN); + + if (qtfs_fullname(cpath, path->dentry, MAX_PATH_LEN) < 0) { + qtfs_err("%s: failed to get fullname", __func__); + goto remote; + } + + pid = is_local_process(cpath); + if (pid > 0) { + sscanf(cpath, "/proc/%s", tmp); + sprintf(local_path, "/local_proc/%s", tmp); + ret = kern_path(local_path, 0, &spath); + if (ret) { + qtfs_err("[%s]: kern_path(%s) failed: %d", __func__, local_path, ret); + goto remote; + } + + ret = vfs_getattr(&spath, stat, req_mask, flags); + path_put(&spath); + if (ret) { + qtfs_err("[%s]: vfs_getattr %s failed: %d", __func__, local_path, ret); + goto remote; + } + qtfs_debug("[%s]: %s success", __func__, local_path); + stat->mode = (stat->mode & ~(S_IFMT)) | S_IFLNK; + kfree(cpath); + kfree(tmp); + kfree(local_path); + return 0; + } + +remote: + if (cpath) + kfree(cpath); + if (tmp) + kfree(tmp); + if (local_path) + kfree(local_path); +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) + return qtfs_getattr(NULL, path, stat, req_mask, flags); +#else + return qtfs_getattr(path, stat, req_mask, flags); +#endif +} diff --git a/qtfs/qtfs/qtfs-mod.c b/qtfs/qtfs/qtfs-mod.c new file mode 100644 index 0000000..f2e1670 --- /dev/null +++ b/qtfs/qtfs/qtfs-mod.c @@ -0,0 +1,316 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) +#include +#endif +#include "conn.h" + +#include "qtfs-mod.h" +#include "syscall.h" +#include "symbol_wrapper.h" + +static struct file_system_type qtfs_fs_type = { + .owner = THIS_MODULE, + .name = QTFS_FSTYPE_NAME, + .mount = qtfs_fs_mount, + .kill_sb = qtfs_kill_sb, +}; +MODULE_ALIAS_FS("qtfs"); +struct kmem_cache *qtfs_inode_priv_cache; +struct task_struct *g_qtfs_epoll_thread = NULL; + +/* + * 转发框架层： + * 1. 调用者先在pvar里预留框架头后，填好自己的私有发送数据。 + 2. 框架负责填充框架头，将整体消息发到对端。 + 3. 等待对端执行完回复。 + 4. 将接收buf的私有数据段首指针返回给调用者，完成文件操作层的通信。 + */ +void *qtfs_remote_run(struct qtfs_conn_var_s *pvar, unsigned int type, unsigned int len) +{ + int ret; + unsigned long retrytimes = 0; + struct qtreq *req = (struct qtreq *)pvar->vec_send.iov_base; + struct qtreq *rsp = (struct qtreq *)pvar->vec_recv.iov_base; + if (req == NULL || type >= QTFS_REQ_INV) { + qtfs_err("qtfs remote run failed, req is NULL type:%u.\n", type); + return NULL; + } + pvar->seq_num++; + req->type = type; + req->len = len; + req->err = 0; + req->seq_num = pvar->seq_num; + + pvar->conn_ops->conn_recv_buff_drop(&pvar->conn_var); + // 调用qtfs_remote_run之前，调用者应该先把消息在iov_base里面封装好 + // 如果不是socket通信，则是在其他通信模式定义的buf里，消息协议统一 + // 都是struct qtreq *xx + // 给server发一个消息 + pvar->vec_send.iov_len = QTFS_MSG_HEAD_LEN + len; + ret = qtfs_conn_send(pvar); + if (ret <= 0) { + qtfs_err("qtfs remote run send failed, ret:%d pvar sendlen:%lu.", ret, pvar->vec_send.iov_len); + qtinfo_senderrinc(req->type); + return NULL; + } + qtinfo_sendinc(type); + + // wait for response +retry: + ret = qtfs_conn_recv_block(pvar); + if (ret == -EAGAIN) + goto retry; + if (ret > 0 && req->seq_num != rsp->seq_num) { + qtinfo_cntinc(QTINF_SEQ_ERR); + qtfs_debug("qtfs remote run recv msg mismatch type:%d, ret:%d pvaridx:%d req:%lu rsp:%lu.", + req->type, ret, pvar->cur_threadidx, req->seq_num, rsp->seq_num); + qtinfo_recvinc(rsp->type); + if (pvar->miss_proc == 0) { + pvar->miss_proc = 1; + qtfs_missmsg_proc(pvar); + pvar->miss_proc = 0; + } + goto retry; + } + if (ret == -ERESTARTSYS) { + if (retrytimes == 0) { + qtinfo_cntinc(QTINF_RESTART_SYS); + qtinfo_recverrinc(req->type); + } + retrytimes++; + msleep(1); + goto retry; + } + if (ret < 0) { + qtfs_err("qtfs remote run error, ret:%d.", ret); + qtinfo_recverrinc(req->type); + return NULL; + } + if (retrytimes > 0) + qtfs_debug("qtfs remote run retry times:%lu.", retrytimes); + qtinfo_recvinc(rsp->type); + + if (rsp->err == QTFS_ERR) { + qtfs_err("qtfs remote run error, req errcode:%d type:%u len:%lu\n", req->err, req->type, req->len); + return NULL; + } + return pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_RECV); +} + +static int qtfs_epoll_thread(void *data) +{ + struct qtfs_conn_var_s *pvar = NULL; + struct qtreq_epollevt *req; + struct qtrsp_epollevt *rsp; + struct qtreq *head; + int ret; + struct inode *inode; + struct file *file; + int i; + +connecting: + while (qtfs_mod_exiting == false) { + pvar = qtfs_epoll_establish_conn(); + if (pvar != NULL) + break; + msleep(500); + } + if (pvar == NULL) { + goto end; + } + qtfs_info("qtfs epoll thread establish a new connection."); + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_RECV); + rsp = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + + // init ack head only once + do { + head = pvar->vec_send.iov_base; + pvar->vec_send.iov_len = QTFS_MSG_HEAD_LEN + sizeof(struct qtrsp_epollevt); + head->type = QTFS_REQ_EPOLL_EVENT; + head->len = sizeof(struct qtrsp_epollevt); + rsp->ret = QTFS_OK; + } while (0); + + while (!kthread_should_stop()) { + ret = qtfs_conn_recv(pvar); + if (ret == -EPIPE || pvar->conn_ops->conn_connected(&pvar->conn_var) == false) + goto connecting; + if (ret < 0 || req->event_nums <= 0 || req->event_nums >= QTFS_MAX_EPEVENTS_NUM) { + continue; + } + qtfs_debug("epoll thread recv %d events.", req->event_nums); + for (i = 0; i < req->event_nums; i++) { + // events[i].data is *file ptr + file = (struct file *)req->events[i].data; + if (IS_ERR_OR_NULL(file)) { + qtfs_err("epoll thread event file invalid!"); + continue; + } + inode = file->f_inode; + // 暂时只支持fifo文件的epoll + if (inode == NULL || !qtfs_support_epoll(inode->i_mode)) { + qtfs_err("epoll thread event file not a fifo."); + continue; + } + do { + struct qtfs_inode_priv *priv = inode->i_private; + __poll_t key; + if (req->events[i].events & EPOLLHUP) + key = EPOLLHUP; + else + key = EPOLLIN | EPOLLRDNORM; + if (priv == NULL) { + qtfs_err("epoll epoll wake up file error, inode priv is invalid."); + WARN_ON(1); + } else { + wake_up_interruptible_sync_poll(&priv->readq, key); + } + } while (0); + } + ret = qtfs_conn_send(pvar); + if (ret < 0) + qtfs_err("conn send failed, ret:%d\n", ret); + } +end: + g_qtfs_epoll_thread = NULL; + return 0; +} + +struct file_operations qtfs_misc_fops = { + .owner=THIS_MODULE, + .unlocked_ioctl = qtfs_misc_ioctl, +}; + +static int __init qtfs_init(void) +{ + int ret; + qtfs_log_init(qtfs_log_level, sizeof(qtfs_log_level)); + + if (qtfs_kallsyms_hack_init() != 0) + return -1; + + ret = register_filesystem(&qtfs_fs_type); + if (ret != 0) { + qtfs_err("QTFS file system register failed, ret:%d.\n", ret); + return -1; + } + qtfs_inode_priv_cache = kmem_cache_create("qtfs_inode_priv", + sizeof(struct qtfs_inode_priv), + 0, + (SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD), + NULL); + if (!qtfs_inode_priv_cache) { + qtfs_err("qtfs inode priv cache create failed.\n"); + ret = -ENOMEM; + goto cache_create_err; + } + if (qtfs_conn_param_init() < 0) { + ret = -ENOMEM; + goto conn_init_err; + } + qtfs_whitelist_initset(); + qtfs_conn_param_init(); + g_qtfs_epoll_thread = kthread_run(qtfs_epoll_thread, NULL, "qtfs_epoll"); + if (IS_ERR_OR_NULL(g_qtfs_epoll_thread)) { + qtfs_err("qtfs epoll thread run failed.\n"); + ret = QTFS_PTR_ERR(g_qtfs_epoll_thread); + goto epoll_thread_err; + } + qtfs_diag_info = (struct qtinfo *)kmalloc(sizeof(struct qtinfo), GFP_KERNEL); + if (qtfs_diag_info == NULL) { + qtfs_err("kmalloc qtfs diag info failed."); + ret = -ENOMEM; + goto diag_malloc_err; + } else { + memset(qtfs_diag_info, 0, sizeof(struct qtinfo)); + } + ret = qtfs_misc_register(); + if (ret) { + goto misc_register_err; + } + ret = qtfs_syscall_replace_start(); + if (ret) { + goto syscall_replace_err; + } + ret = qtfs_syscall_init(); + if (ret) { + goto syscall_init_err; + } + ret = qtfs_utils_register(); + if (ret) { + goto utils_register_err; + } + qtfs_info("QTFS file system register success!\n"); + return 0; +utils_register_err: + qtfs_syscall_fini(); +syscall_init_err: + qtfs_syscall_replace_stop(); +syscall_replace_err: + qtfs_misc_destroy(); +misc_register_err: + kfree(qtfs_diag_info); +diag_malloc_err: + kthread_stop(g_qtfs_epoll_thread); +epoll_thread_err: + qtfs_conn_param_fini(); +conn_init_err: + kmem_cache_destroy(qtfs_inode_priv_cache); +cache_create_err: + unregister_filesystem(&qtfs_fs_type); + return ret; +} + +static void __exit qtfs_exit(void) +{ + int ret; + qtfs_mod_exiting = true; + + if (g_qtfs_epoll_thread) { + kthread_stop(g_qtfs_epoll_thread); + } + + qtfs_conn_param_fini(); + qtfs_misc_destroy(); + if (qtfs_epoll_var != NULL) { + qtfs_epoll_cut_conn(qtfs_epoll_var); + qtfs_epoll_var->conn_ops->conn_var_fini(qtfs_epoll_var); + kfree(qtfs_epoll_var); + qtfs_epoll_var = NULL; + } + qtfs_whitelist_exit(); + + kfree(qtfs_diag_info); + qtfs_diag_info = NULL; + qtfs_syscall_fini(); + + ret = unregister_filesystem(&qtfs_fs_type); + if (ret != 0) { + qtfs_err("QTFS file system unregister failed, ret:%d.\n", ret); + } + qtfs_utils_destroy(); + kmem_cache_destroy(qtfs_inode_priv_cache); + qtfs_syscall_replace_stop(); + qtfs_info("QTFS file system unregister success!\n"); + return; +} + +module_init(qtfs_init); +module_exit(qtfs_exit); +MODULE_AUTHOR("liqiang64@huawei.com"); +MODULE_LICENSE("GPL"); diff --git a/qtfs/qtfs/qtfs-mod.h b/qtfs/qtfs/qtfs-mod.h new file mode 100644 index 0000000..3939e0f --- /dev/null +++ b/qtfs/qtfs/qtfs-mod.h @@ -0,0 +1,196 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __QTFS_INCLUDE_H__ +#define __QTFS_INCLUDE_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "comm.h" +#include "log.h" +#include "req.h" + + +#define QTFS_MAXLEN 8 +#define QTFS_MAX_FILES 32 +#define QTFS_MAX_BLOCKSIZE 512 + +#define QTFS_FSTYPE_NAME "qtfs" + +extern struct kmem_cache *qtfs_inode_priv_cache; + +struct private_data { + int fd; +}; + +struct qtfs_inode_priv { + unsigned int files; + wait_queue_head_t readq; + wait_queue_head_t writeq; +}; + +enum { + QTFS_ROOT_INO = 1, + QTFS_IPC_INIT_INO = 0xEFFFFFFFU, + QTFS_UTS_INIT_INO = 0xEFFFFFFEU, + QTFS_USER_INIT_INO = 0xEFFFFFFDU, + QTFS_PID_INIT_INO = 0xEFFFFFFCU, + QTFS_CGROUP_INIT_INO = 0xEFFFFFFBU, + QTFS_TIME_INIT_INO = 0xEFFFFFFAU, + QTFS_IMA_INIT_INO = 0xEFFFFFF9U, +}; + +struct qtfs_inode { + mode_t mode; + uint64_t i_no; + uint64_t d_no; + char *peer_path; + union { + uint64_t file_size; + uint64_t dir_childrens; + }; + struct list_head entry; +}; + +struct qtfs_fs_info { + char peer_path[NAME_MAX]; + char *mnt_path; + + enum qtfs_type type; +}; + +struct qtfs_dir_entry { + struct list_head node; + char filename[NAME_MAX]; + struct qtfs_inode *priv; +}; + +struct qtfs_file_blk { + uint8_t busy; + mode_t mode; + uint8_t idx; + + union { + uint8_t file_size; + uint8_t dir_children; + }; + char data[0]; +}; + +struct qtmiss_ops { + int type; + // return int is output len. + int (*misshandle) (struct qtreq *); + char str[32]; +}; + +static inline int qtfs_fullname(char *fullname, struct dentry *d, size_t buflen) +{ + struct qtfs_fs_info *fsinfo = NULL; + int len = 0; + char *name = NULL; + char *ret = NULL; + + if (!d) { + qtfs_info("%s: get dentry fullname NULL\n", __func__); + return -1; + } + if (buflen < MAX_PATH_LEN) { + qtfs_err("%s: failed to get fullname dure to small buflen:%lu\n", __func__, buflen); + return -1; + } + + name = __getname(); + if (!name) { + return -1; + } + ret = dentry_path_raw(d, name, MAX_PATH_LEN); + if (IS_ERR_OR_NULL(ret)) { + qtfs_err("qtfs fullname failed:%ld\n", QTFS_PTR_ERR(ret)); + __putname(name); + return -1; + } + + if (d && d->d_sb && d->d_sb->s_fs_info) { + fsinfo = d->d_sb->s_fs_info; + } else { + qtfs_err("%s: failed to get private fs_info\n", __func__); + __putname(name); + return -1; + } + if (strcmp(fsinfo->peer_path, "/")) { + /* if peer_path is not root '/' */ + len = strlcpy(fullname, fsinfo->peer_path, MAX_PATH_LEN); + } + if (len + strlen(ret) >= MAX_PATH_LEN - 1) { + qtfs_err("qtfs fullname may reach max len:%d reallen:%ld path:%s", len, strlen(fullname), fullname); + __putname(name); + return -1; + } + len += strlcpy(&fullname[len], ret, MAX_PATH_LEN - len); + if (strcmp(fullname, "/")) { + if (fullname[strlen(fullname) - 1] == '/') + fullname[strlen(fullname) - 1] = '\0'; + } + __putname(name); + return len; +} + +#define QTFS_FULLNAME(fullname, d, buflen) \ + if (qtfs_fullname(fullname, d, buflen)<0) { \ + qtfs_err("qtfs fullname failed\n"); \ + qtfs_conn_put_param(pvar); \ + return -EINVAL; \ + } + +extern const struct xattr_handler qtfs_xattr_user_handler; +extern const struct xattr_handler qtfs_xattr_trusted_handler; +extern const struct xattr_handler qtfs_xattr_security_handler; +extern const struct xattr_handler qtfs_xattr_hurd_handler; +extern struct qtinfo *qtfs_diag_info; +extern int qtfs_mod_exiting; + +void qtfs_kill_sb(struct super_block *sb); +struct dentry *qtfs_fs_mount(struct file_system_type *fs_type, + int flags, const char *dev_name, + void *data); +void *qtfs_remote_run(struct qtfs_conn_var_s *pvar, unsigned int type, unsigned int len); +int qtfs_misc_register(void); +void qtfs_misc_destroy(void); +void qtfs_whitelist_exit(void); +long qtfs_misc_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +int qtfs_missmsg_proc(struct qtfs_conn_var_s *pvar); +int qtfs_utils_register(void); +void qtfs_utils_destroy(void); +void qtfs_whitelist_clearall(void); +void qtfs_whitelist_initset(void); + +#endif + diff --git a/qtfs/qtfs/sb.c b/qtfs/qtfs/sb.c new file mode 100644 index 0000000..0ead9e1 --- /dev/null +++ b/qtfs/qtfs/sb.c @@ -0,0 +1,1668 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "conn.h" +#include "qtfs-mod.h" +#include "req.h" +#include "log.h" +#include "ops.h" +#include "symbol_wrapper.h" + +#define CURRENT_TIME(inode) (current_time(inode)) +static struct inode_operations qtfs_inode_ops; +static struct inode_operations qtfs_symlink_inode_ops; +struct inode *qtfs_iget(struct super_block *sb, struct inode_info *ii); +extern ssize_t qtfs_xattr_list(struct dentry *dentry, char *buffer, size_t buffer_size); +int qtfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_statfs *req; + struct qtrsp_statfs *rsp; + if (!pvar) { + qtfs_err("Failed to get qtfs sock var"); + return -EINVAL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + rsp = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_RECV); + + QTFS_FULLNAME(req->path, dentry, sizeof(req->path)); + rsp = qtfs_remote_run(pvar, QTFS_REQ_STATFS, QTFS_SEND_SIZE(struct qtreq_statfs, req->path)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return -EINVAL; + } + if (rsp->ret == QTFS_ERR) { + int ret = rsp->errno; + qtfs_err("qtfs statfs failed. %d", rsp->errno); + qtfs_conn_put_param(pvar); + return ret; + } + qtfs_info("%s: get path %s\n", __func__, req->path); + memcpy(buf, &(rsp->kstat), sizeof(struct kstatfs)); + qtfs_conn_put_param(pvar); + return 0; +} + +static void qtfs_free_inode(struct inode *inode) +{ + if (inode->i_private) { + kmem_cache_free(qtfs_inode_priv_cache, inode->i_private); + inode->i_private = NULL; + } + free_inode_nonrcu(inode); + return; +} + +static const struct super_operations qtfs_ops = { + .statfs = qtfs_statfs, + .free_inode = qtfs_free_inode, +}; + +static inline struct qtfs_fs_info *qtfs_priv_byinode(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + return sb->s_fs_info; +} + +static inline char *qtfs_mountpoint_path_init(struct dentry *dentry, struct path *path, char *mnt_file) +{ + char *name = NULL; + char *ret; + char *mnt_point; + size_t len; + struct qtfs_fs_info *fsinfo = qtfs_priv_byinode(d_inode(dentry)); + if (fsinfo && fsinfo->mnt_path) { + return fsinfo->mnt_path; + } + name = __getname(); + if (!name) { + return ERR_PTR(-ENOMEM); + } + path_get(path); + ret = qtfs_kern_syms.d_absolute_path(path, name, MAX_PATH_LEN); + qtfs_debug("mntfile:%s absolute:%s", mnt_file, ret); + if (IS_ERR_OR_NULL(ret)) { + qtfs_err("d_absolute_path failed:%ld", QTFS_PTR_ERR(ret)); + } else { + if (strcmp(mnt_file, "/")) { + mnt_point = strstr(ret, mnt_file); + qtfs_info("mnt point:%s", mnt_point); + if (mnt_point) { + *mnt_point = '\0'; + } else { + qtfs_err("Failed to get mount root path"); + } + } + len = strlen(ret); + if (len == 0) { + qtfs_err("mount path len invalid."); + goto end; + } + fsinfo->mnt_path = (char *)kmalloc(len + 1, GFP_KERNEL); + if (fsinfo->mnt_path) { + strlcpy(fsinfo->mnt_path, ret, len + 1); + } + qtfs_debug("d_absolute_path get mnt path:%s", fsinfo->mnt_path); + } +end: + path_put(path); + __putname(name); + return fsinfo->mnt_path; +} + +int qtfs_readdir(struct file *filp, struct dir_context *ctx) +{ + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_readdir *req; + struct qtrsp_readdir *rsp; + struct qtfs_dirent64 *dirent = NULL; + int idx; + int ret; + int namelen; + int dircnt; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var"); + return -EINVAL; + } + + if (ctx->pos == -1) { + qtfs_conn_put_param(pvar); + return -ENOENT; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + rsp = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_RECV); + QTFS_FULLNAME(req->path, filp->f_path.dentry, sizeof(req->path)); + req->count = sizeof(rsp->dirent); + req->pos = ctx->pos; + + rsp = qtfs_remote_run(pvar, QTFS_REQ_READDIR, QTFS_SEND_SIZE(struct qtreq_readdir, req->path)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(rsp); + } + if (rsp->d.ret == QTFS_ERR || rsp->d.vldcnt < 0 || rsp->d.pos < 0) { + qtfs_err("qtfs readdir failed."); + qtfs_conn_put_param(pvar); + return -EFAULT; + } + + idx = 0; + dircnt = rsp->d.vldcnt; + while (dircnt-- > 0) { + if (idx >= sizeof(rsp->dirent)) { + qtfs_err("invalid idx:%d", idx); + break; + } + dirent = (struct qtfs_dirent64 *)&rsp->dirent[idx]; + namelen = strlen(dirent->d_name); + ret = dir_emit(ctx, dirent->d_name, namelen, + dirent->d_ino, dirent->d_type); + idx += dirent->d_reclen; + qtfs_debug("qtfs readdir direntoff:0x%lx name:<%s>, ret:%d, reclen:%u namelen:%d, ino:%llu type:%d", + (void *)dirent - (void *)rsp->dirent, dirent->d_name, ret, dirent->d_reclen, namelen, dirent->d_ino, dirent->d_type); + } + + ctx->pos = (rsp->d.over) ? -1 : rsp->d.pos; + qtfs_info("qtfs readdir<%s> success ret:%d vldcnt:%d over:%d pos:%lld.", + req->path, rsp->d.ret, rsp->d.vldcnt, rsp->d.over, ctx->pos); + qtfs_conn_put_param(pvar); + return 0; +} + +int qtfs_open(struct inode *inode, struct file *file) +{ + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_open *req; + struct qtrsp_open *rsp; + struct private_data *data = NULL; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var"); + return -EINVAL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, file->f_path.dentry, sizeof(req->path)); + + req->flags = file->f_flags; + req->mode = file->f_mode; + rsp = qtfs_remote_run(pvar, QTFS_REQ_OPEN, QTFS_SEND_SIZE(struct qtreq_open, req->path)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + qtfs_err("qtfs open:%s failed, f_mode:%o flag:%x", req->path, file->f_mode, file->f_flags); + return -EINVAL; + } + + if (rsp->ret == QTFS_ERR) { + int err = rsp->fd; + if (rsp->fd != -ENOENT) { + qtfs_err("qtfs_open failed with %d ret:%d", rsp->fd, rsp->ret); + } else { + qtfs_info("qtfs_open file %s failed, not exist.", req->path); + } + qtfs_conn_put_param(pvar); + return err; + } + qtfs_info("qtfs open:%s success, f_mode:%o flag:%x, fd:%d", req->path, file->f_mode, file->f_flags, rsp->fd); + + data = (struct private_data *)kmalloc(sizeof(struct private_data), GFP_KERNEL); + if (IS_ERR_OR_NULL(data)) { + qtfs_err("qtfs_open alloc private_data failed: %ld", QTFS_PTR_ERR(data)); + qtfs_conn_put_param(pvar); + return -ENOMEM; + } + + data->fd = rsp->fd; + WARN_ON(file->private_data); + file->private_data = data; + qtfs_conn_put_param(pvar); + + return 0; +} + +int qtfs_dir_open(struct inode *inode, struct file *file) +{ + qtfs_info("qtfs dir open enter: %s.", file->f_path.dentry->d_iname); + return 0; +} + +int qtfs_dir_release(struct inode *inode, struct file *file) +{ + qtfs_info("qtfs dir release enter: %s.", file->f_path.dentry->d_iname); + return 0; +} + +int qtfs_release(struct inode *inode, struct file *file) +{ + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_close *req; + struct qtrsp_close *rsp; + struct private_data *private = NULL; + int ret; + + if (pvar == NULL) { + qtfs_err("qtfs release pvar invalid."); + return -EFAULT; + } + + if (IS_ERR_OR_NULL(file)) { + qtfs_err("qtfs release: invalid file: 0x%llx", (__u64)file); + qtfs_conn_put_param(pvar); + return -EINVAL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + private = (struct private_data *)file->private_data; + + if (IS_ERR_OR_NULL(private)) { + qtfs_err("qtfs_close(%s): invalid private_data pointer:%ld", file->f_path.dentry->d_iname, QTFS_PTR_ERR(private)); + WARN_ON(1); + qtfs_conn_put_param(pvar); + return -EFAULT; + } + req->fd = private->fd; + rsp = qtfs_remote_run(pvar, QTFS_REQ_CLOSE, sizeof(struct qtreq_close)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_err("qtfs release fd:%d failed, rsp is invalid.", req->fd); + ret = QTFS_PTR_ERR(rsp); + goto end; + } + qtfs_info("qtfs release success fd:%d ret:%d %s", req->fd, rsp->ret, (rsp->ret == QTFS_ERR) ? "failed" : "success"); + ret = rsp->ret; +end: + qtfs_conn_put_param(pvar); + kfree(file->private_data); + file->private_data = NULL; + return ret; +} + +ssize_t qtfs_readiter(struct kiocb *kio, struct iov_iter *iov) +{ + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_readiter *req; + struct qtrsp_readiter *rsp; + int reqlen; + size_t leftlen = iov_iter_count(iov); + size_t allcnt = leftlen; + size_t tocnt = 0; + ssize_t ret; + struct private_data *private = NULL; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var"); + return -EINVAL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + + private = (struct private_data *)kio->ki_filp->private_data; + if (IS_ERR_OR_NULL(private)) { + qtfs_err("qtfs_readiter(%s): invalid private_data pointer:%ld", kio->ki_filp->f_path.dentry->d_iname, QTFS_PTR_ERR(private)); + qtfs_conn_put_param(pvar); + return -ENOMEM; + } + + req->fd = private->fd; + if (req->fd <= 0) { + qtfs_err("qtfs_readiter: invalid file(%d)", req->fd); + qtfs_conn_put_param(pvar); + return -EINVAL; + } + reqlen = sizeof(struct qtreq_readiter); + + do { + req->len = leftlen; + req->pos = kio->ki_pos; + rsp = qtfs_remote_run(pvar, QTFS_REQ_READITER, reqlen); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(rsp); + } + if (rsp->d.ret == QTFS_ERR || rsp->d.len <= 0 || rsp->d.len > leftlen) { + if (rsp->d.len != 0) + qtfs_info("qtfs readiter error: %ld.", rsp->d.len); + ret = (rsp->d.len > leftlen) ? leftlen : (ssize_t)rsp->d.len; + qtfs_conn_put_param(pvar); + return (ret > 0) ? allcnt - leftlen + ret : allcnt - leftlen; + } + tocnt = copy_to_iter(rsp->readbuf, rsp->d.len, iov); + if (rsp->d.len != tocnt) { + qtfs_err("copy to iter failed, errno:%ld", tocnt); + qtfs_conn_put_param(pvar); + return allcnt - leftlen + tocnt; + } + + leftlen -= rsp->d.len; + kio->ki_pos += rsp->d.len; + } while (leftlen > 0 && rsp->d.end == 0); + qtfs_info("qtfs readiter over, leftlen:%lu, reqlen:%lu, fullname:<%s>, ino:%lu, pos:%lld, iovcnt:%lu\n", leftlen, + req->len, kio->ki_filp->f_path.dentry->d_iname, kio->ki_filp->f_inode->i_ino, kio->ki_pos, iov_iter_count(iov)); + + qtfs_conn_put_param(pvar); + return allcnt - leftlen; +} + +ssize_t qtfs_writeiter(struct kiocb *kio, struct iov_iter *iov) +{ + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_write *req; + struct qtrsp_write *rsp; + char *wrbuf = NULL; + int wrbuflen; + int maxbuflen; + size_t len = iov_iter_count(iov); + size_t leftlen = len; + struct private_data *private = NULL; + ssize_t ret; + struct file *filp; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var."); + return -EINVAL; + } + if (len <= 0) { + qtfs_conn_put_param(pvar); + return len; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + filp = kio->ki_filp; + private = (struct private_data *)filp->private_data; + if (IS_ERR_OR_NULL(private)) { + qtfs_err("qtfs_write(%s): invalid private_data pointer:%ld", filp->f_path.dentry->d_iname, QTFS_PTR_ERR(private)); + qtfs_conn_put_param(pvar); + return -ENOMEM; + } + + req->d.fd = private->fd; + if (req->d.fd < 0) { + qtfs_err("qtfs_write: invalid file(%d)", req->d.fd); + qtfs_conn_put_param(pvar); + return -EINVAL; + } + req->d.mode = filp->f_mode; + req->d.flags = filp->f_flags; + + wrbuf = req->path_buf; + maxbuflen = sizeof(req->path_buf); + do { + req->d.total_len = len; + wrbuflen = (leftlen >= maxbuflen) ? (maxbuflen - 1) : leftlen; + req->d.buflen = wrbuflen; + req->d.pos = kio->ki_pos; + if (copy_from_iter(wrbuf, wrbuflen, iov) == 0) { + qtfs_err("qtfs write copy from iter failed, len:%d.", wrbuflen); + break; + } + rsp = qtfs_remote_run(pvar, QTFS_REQ_WRITE, sizeof(struct qtreq_write) - sizeof(req->path_buf) + wrbuflen); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(rsp); + } + if (rsp->len > wrbuflen) { + qtfs_err("qtfs write recv error packet, len:%ld writelen:%d", rsp->len, wrbuflen); + break; + } + if (rsp->ret == QTFS_ERR || rsp->len <= 0) { + qtfs_err("qtfs write remote error, errno:%ld, leftlen:%lu.", rsp->len, leftlen); + if (rsp->len > 0) { + kio->ki_pos += rsp->len; + leftlen -= rsp->len; + break; + } + ret = rsp->len; + qtfs_conn_put_param(pvar); + return (ret > 0) ? len - leftlen + ret : len - leftlen; + } + if (rsp->len != wrbuflen) { + iov->count -= (rsp->len - wrbuflen); + iov->iov_offset += (rsp->len - wrbuflen); + } + kio->ki_pos += rsp->len; + leftlen -= rsp->len; + } while (leftlen > 0); + + do { + struct inode *inode = kio->ki_filp->f_inode; + struct qtfs_inode_priv *priv = inode->i_private; + if (S_ISFIFO(inode->i_mode)) + wake_up_interruptible_sync_poll(&priv->readq, EPOLLIN | EPOLLRDNORM); + if (S_ISCHR(inode->i_mode)) { + wake_up_interruptible_poll(&priv->readq, EPOLLIN); + qtfs_err("writeiter file:%s char:<%s> wakup poll.", filp->f_path.dentry->d_iname, req->path_buf); + } + } while (0); + qtfs_info("qtfs write %s over, leftlen:%lu.", filp->f_path.dentry->d_iname, leftlen); + qtfs_conn_put_param(pvar); + return len - leftlen; +} + +loff_t qtfs_llseek(struct file *file, loff_t off, int whence) +{ + struct qtfs_conn_var_s *pvar = NULL; + struct qtreq_llseek *req; + struct qtrsp_llseek *rsp; + off_t ret; + struct private_data *priv = NULL; + + qtfs_info("qtfs llseek off:%lld, whence:%d cur pos:%lld.", off, whence, file->f_pos); + + if (off == 0 && whence == SEEK_CUR) { + return file->f_pos; + } + pvar = qtfs_conn_get_param(); + if (!pvar) { + qtfs_err("Failed to get qtfs sock var."); + return -EINVAL; + } + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + + priv = (struct private_data *)file->private_data; + req->off = off; + req->whence = whence; + req->fd = priv->fd; + rsp = qtfs_remote_run(pvar, QTFS_REQ_LLSEEK, sizeof(struct qtreq_llseek)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + qtfs_err("Failed to remote run llseek."); + return QTFS_PTR_ERR(rsp); + } + if (rsp->ret != QTFS_OK) { + ret = rsp->off; + qtfs_conn_put_param(pvar); + return ret; + } + file->f_pos = rsp->off; + ret = rsp->off; + qtfs_conn_put_param(pvar); + qtfs_info("qtfs llseek successed, cur seek pos:%ld.", ret); + return ret; +} + +static void qtfs_vma_close(struct vm_area_struct *vma) +{ + qtfs_info("qtfs vma close enter."); + filemap_write_and_wait(vma->vm_file->f_mapping); +} + +static vm_fault_t qtfs_vm_fault(struct vm_fault *vmf) +{ + vm_fault_t ret = filemap_fault(vmf); + + qtfs_info("qtfs vm ops fault enter, filemap fault:0x%x, pgoff:%lu.", ret, vmf->pgoff); + return ret; +} + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) +static vm_fault_t qtfs_map_pages(struct vm_fault *vmf, + pgoff_t start_pgoff, pgoff_t end_pgoff) + { + qtfs_info("qtfs map pages enter, pgoff:%lu start:%lu end:%lu.", vmf->pgoff, start_pgoff, end_pgoff); + return filemap_map_pages(vmf, start_pgoff, end_pgoff); + } +#else +static void qtfs_map_pages(struct vm_fault *vmf, + pgoff_t start_pgoff, pgoff_t end_pgoff) +{ + qtfs_info("qtfs map pages enter, pgoff:%lu start:%lu end:%lu.", vmf->pgoff, start_pgoff, end_pgoff); + + filemap_map_pages(vmf, start_pgoff, end_pgoff); + return; +} +#endif + +static vm_fault_t qtfs_page_mkwrite(struct vm_fault *vmf) +{ + qtfs_info("qtfs page mkwrite enter."); + return filemap_page_mkwrite(vmf); +} + +static const struct vm_operations_struct qtfs_file_vm_ops = { + .fault = qtfs_vm_fault, + .map_pages = qtfs_map_pages, + .close = qtfs_vma_close, + .page_mkwrite = qtfs_page_mkwrite, +}; + +int qtfs_mmap(struct file *file, struct vm_area_struct *vma) +{ + qtfs_info("qtfs mmap enter."); + + if (IS_DAX(file_inode(file))) { + qtfs_info("qtfs mmap is dax mmap."); + } + file_accessed(file); + vma->vm_ops = &qtfs_file_vm_ops; + return 0; +} + +int qtfs_fsync(struct file *file, loff_t start, loff_t end, int datasync) +{ + qtfs_info("qtfs fsync enter."); + return 0; +} + +long qtfs_do_ioctl(struct file *filp, unsigned int cmd, unsigned long arg, unsigned int size, int argtype) +{ + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_ioctl *req; + struct qtrsp_ioctl *rsp; + unsigned int len = 0; + int ret = -EINVAL; + struct private_data *priv = NULL; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var"); + return -EINVAL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + rsp = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_RECV); + if (size >= sizeof(req->path)) { + qtfs_err("do ioctl failed, size:%u too big:%lu", size, sizeof(req->path)); + qtfs_conn_put_param(pvar); + return -EINVAL; + } + + priv = (struct private_data *)filp->private_data; + req->d.fd = priv->fd; + req->d.argtype = argtype; + req->d.cmd = cmd; + if (argtype) { + req->d.arg = arg; + len = sizeof(struct qtreq_ioctl) - sizeof(req->path); + } else if (size > 0) { + ret = copy_from_user(req->path, (char __user *)arg, size); + if (ret) { + qtfs_err("%s: copy_from_user, size %u failed.", __func__, size); + ret = -EFAULT; + goto out; + } + len = sizeof(struct qtreq_ioctl) - sizeof(req->path) + size; + req->d.size = size; + } else { + len = sizeof(struct qtreq_ioctl) - sizeof(req->path); + } + + rsp = qtfs_remote_run(pvar, QTFS_REQ_IOCTL, len); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(rsp); + } + if (rsp->ret == QTFS_ERR) { + qtfs_err("qtfs ioctl cmd:0x%x failed. %d", cmd, rsp->errno); + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; + } + + qtfs_info("qtfs do ioctl cmd:0x%x success, path: %s size:%u, rsp size:%u", cmd, req->path, size, rsp->size); + ret = rsp->errno; + if (rsp->size > sizeof(rsp->buf) || + (rsp->size > 0 && copy_to_user((char __user *)arg, rsp->buf, size))) { + qtfs_err("copy to user failed"); + ret = -EFAULT; + } +out: + qtfs_conn_put_param(pvar); + return (long)ret; +} + +#define QTFS_IOCTL_CASE_WITH_BREAK(size, argtype)\ + {\ + ret = qtfs_do_ioctl(filp, cmd, arg, size, argtype);\ + break;\ + } +long qtfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + long ret; + switch(cmd) { + // all case of size 0 type 0 enter here + case FS_IOC_FSGETXATTR: + case TCGETS: + QTFS_IOCTL_CASE_WITH_BREAK(0, 0); + // all case of size 0 type 1 enter here + case TUNSETPERSIST: + QTFS_IOCTL_CASE_WITH_BREAK(0, 1); + case FS_IOC_FSSETXATTR: + QTFS_IOCTL_CASE_WITH_BREAK(sizeof(struct fsxattr), 0); + case TCSETS: + QTFS_IOCTL_CASE_WITH_BREAK(sizeof(struct ktermios), 0); + case TUNSETIFF: + case SIOCGIFHWADDR: + case SIOCADDMULTI: + case SIOCBRADDIF: + case SIOCBRDELIF: + case TUNGETIFF: + case SIOCDELMULTI: + case SIOCDEVPRIVATE: + case SIOCETHTOOL: + case SIOCGIFADDR: + case SIOCGIFFLAGS: + case SIOCGIFINDEX: + case SIOCGIFMTU: + case SIOCSIFFLAGS: + case SIOCSIFHWADDR: + case SIOCSIFMTU: + case SIOCSIFNAME: + QTFS_IOCTL_CASE_WITH_BREAK(sizeof(struct ifreq), 0); + case SIOCBRADDBR: + case SIOCBRDELBR: + QTFS_IOCTL_CASE_WITH_BREAK(IFNAMSIZ, 0); + case SIOCGIFVLAN: + QTFS_IOCTL_CASE_WITH_BREAK(sizeof(struct vlan_ioctl_args), 0); + default: { + char *fullname = kmalloc(MAX_PATH_LEN, GFP_KERNEL); + if (!fullname) + return -ENOMEM; + memset(fullname, 0, MAX_PATH_LEN); + qtfs_fullname(fullname, filp->f_path.dentry, MAX_PATH_LEN); + qtfs_err("qtfs ioctl get not support cmd:%d file:%s", cmd, fullname); + kfree(fullname); + return -EOPNOTSUPP; + } + } + return ret; +} + +loff_t qtfs_dir_file_llseek(struct file *file, loff_t offset, int whence) +{ + qtfs_info("qtfs generic file llseek: %s.", file->f_path.dentry->d_iname); + return generic_file_llseek(file, offset, whence); +} + +ssize_t qtfs_dir_read_dir(struct file *filp, char __user *buf, size_t siz, loff_t *ppos) +{ + qtfs_err("qtfs generic read dir: %s.", filp->f_path.dentry->d_iname); + return generic_read_dir(filp, buf, siz, ppos); +} + +static struct file_operations qtfs_dir_ops = { + .owner = THIS_MODULE, + .iterate_shared = qtfs_readdir, + .unlocked_ioctl = qtfs_ioctl, + .open = qtfs_dir_open, + .release = qtfs_dir_release, + .llseek = qtfs_dir_file_llseek, + .read = qtfs_dir_read_dir, +}; + +static struct file_operations qtfs_file_ops = { + .read_iter = qtfs_readiter, + .write_iter = qtfs_writeiter, + .open = qtfs_open, + .release = qtfs_release, + .mmap = qtfs_mmap, + .llseek = qtfs_llseek, + .fsync = qtfs_fsync, + .unlocked_ioctl = qtfs_ioctl, +}; + +static int qtfs_readpage(struct file *file, struct page *page) +{ + void *kaddr = NULL; + loff_t offset = page->index << PAGE_SHIFT; + qtfs_info("qtfs readpage enter, page pos:%lld.", offset); + + kaddr = kmap_atomic(page); + kernel_read(file, kaddr, PAGE_SIZE, &offset); + flush_dcache_page(page); + kunmap_atomic(kaddr); + SetPageUptodate(page); + unlock_page(page); + + return 0; +} + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0)) +static int qtfs_read_folio(struct file *file, struct folio *folio) +{ + struct page *page = &folio->page; + qtfs_readpage(file, page); + + return 0; +} +#endif + +#ifndef KVER_4_19 +static struct page **qtfs_alloc_pages(unsigned int nr) +{ + struct page **pages = kzalloc(nr * (sizeof(struct page *)), GFP_KERNEL); + if (pages == NULL) { + qtfs_err("qtfs alloc pages failed."); + return NULL; + } + return pages; +} + +static void qtfs_free_pages(struct page **pages) +{ + kfree(pages); +} + +static void qtfs_readahead(struct readahead_control *rac) +{ + int i; + unsigned int nr_pages = readahead_count(rac); + struct page **pages = qtfs_alloc_pages(nr_pages); + qtfs_info("qtfs readahead."); + + nr_pages = __readahead_batch(rac, pages, nr_pages); + + for (i = 0; i < nr_pages; i++) { + qtfs_readpage(rac->file, pages[i]); + } + qtfs_free_pages(pages); + return; +} +#endif + +static int qtfs_writepage(struct page *page, struct writeback_control *wbc) +{ + qtfs_info("qtfs write page."); + return 0; +} + +static int qtfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + qtfs_info("qtfs write pages."); + return 0; +} + +static ssize_t qtfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +{ + qtfs_info("qtfs direct IO."); + return 0; +} + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)) +static bool qtfs_dirty_folio(struct address_space *mapping, struct folio *folio) +{ + qtfs_info("qtfs set page dirty."); + return filemap_dirty_folio(mapping, folio); +} +#else +static int qtfs_setpagedirty(struct page *page) +{ + qtfs_info("qtfs set page dirty."); + __set_page_dirty_nobuffers(page); + return 0; +} +#endif + +static const struct address_space_operations qtfs_aops = { +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0)) + .read_folio = qtfs_read_folio, +#else + .readpage = qtfs_readpage, +#endif +#ifndef KVER_4_19 + .readahead = qtfs_readahead, +#endif + .writepage = qtfs_writepage, + .writepages = qtfs_writepages, + .direct_IO = qtfs_direct_IO, +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)) + .dirty_folio = qtfs_dirty_folio, +#else + .set_page_dirty = qtfs_setpagedirty, +#endif +}; + +int qtfs_new_entry(struct inode *inode, struct dentry *dentry) +{ + struct dentry *d = NULL; + + if (!inode) + return -ENOMEM; + + d_drop(dentry); + d = d_splice_alias(inode, dentry); + if (IS_ERR(d)) { + return PTR_ERR(d); + } + if (d) { + if (d->d_inode && S_ISDIR(d->d_inode->i_mode)) + d->d_time = jiffies; + dput(d); + } + return 0; +} + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) +int qtfs_mkdir(struct mnt_idmap *mnt_userns, struct inode *dir, struct dentry *dentry, umode_t mode) +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) +int qtfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, struct dentry *dentry, umode_t mode) +#else +int qtfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +#endif +{ + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_mkdir *req = NULL; + struct qtrsp_mkdir *rsp = NULL; + int ret; + struct inode *inode; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var."); + return -EINVAL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, dentry, sizeof(req->path)); + + req->mode = mode; + rsp = qtfs_remote_run(pvar, QTFS_REQ_MKDIR, QTFS_SEND_SIZE(struct qtreq_mkdir, req->path)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(rsp); + } + if (rsp->ret == QTFS_ERR) { + qtfs_err("qtfs mkdir failed %d.", rsp->errno); + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; + } + inode = qtfs_iget(dentry->d_sb, &(rsp->inode_info)); + ret = qtfs_new_entry(inode, dentry); + qtfs_info("mkdir path:%s success.", req->path); + qtfs_conn_put_param(pvar); + return ret; +} + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) +int qtfs_create(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, umode_t mode, bool excl) +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) +int qtfs_create(struct user_namespace *mnt_userns, struct inode *dir, struct dentry *dentry, umode_t mode, bool excl) +#else +int qtfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool excl) +#endif +{ + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_icreate *req; + struct qtrsp_icreate *rsp; + struct inode *inode; + int ret = 0; + int ret2 = 0; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var."); + return -EINVAL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, dentry, sizeof(req->path)); + + req->mode = mode; + req->excl = excl; + rsp = qtfs_remote_run(pvar, QTFS_REQ_ICREATE, QTFS_SEND_SIZE(struct qtreq_icreate, req->path)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(rsp); + } + + if (rsp->ret == QTFS_ERR) { + ret = rsp->errno; + qtfs_err("qtfs icreate failed %d.", rsp->errno); + qtfs_conn_put_param(pvar); + return ret; + } + ret = rsp->errno; + inode = qtfs_iget(dentry->d_sb, &(rsp->inode_info)); + ret2 = qtfs_new_entry(inode, dentry); + + qtfs_info("qtfs icreate get ret:%d, mode:%ho.", rsp->errno, rsp->inode_info.mode); + qtfs_conn_put_param(pvar); + return ret ? ret : ret2; +} + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) +int qtfs_mknod(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) +int qtfs_mknod(struct user_namespace *mnt_userns, struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) +#else +int qtfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) +#endif +{ + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_mknod *req; + struct qtrsp_mknod *rsp; + struct inode *inode; + int ret = 0; + int ret2 = 0; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return -EINVAL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, dentry, sizeof(req->path)); + + req->mode = mode; + req->dev = dev; + rsp = qtfs_remote_run(pvar, QTFS_REQ_MKNOD, sizeof(struct qtreq_mknod) - sizeof(req->path) + strlen(req->path)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(rsp); + } + if (rsp->ret == QTFS_ERR) { + qtfs_err("qtfs mknod failed %d.", rsp->errno); + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; + } + ret = rsp->errno; + qtfs_info("qtfs mknod success, path:<%s>.\n", req->path); + inode = qtfs_iget(dentry->d_sb, &(rsp->inode_info)); + ret2 = qtfs_new_entry(inode, dentry); + qtfs_conn_put_param(pvar); + return ret ? ret : ret2; +} + +static void qtfs_inode_priv_alloc(struct inode *inode) +{ + struct qtfs_inode_priv *priv = kmem_cache_alloc(qtfs_inode_priv_cache, GFP_KERNEL); + if (priv == NULL) { + qtfs_err("qtfs inode priv alloc kmem cache alloc failed."); + return; + } + inode->i_private = priv; + priv->files = 0; + init_waitqueue_head(&priv->readq); + init_waitqueue_head(&priv->writeq); + return; +} + +static void qtfs_init_inode(struct super_block *sb, struct inode *inode, struct inode_info *ii) +{ + inode->i_sb = sb; + inode->i_mode = ii->mode; + inode->i_ino = ii->i_ino; + inode->i_size = ii->i_size; + inode->i_atime = ii->atime; + inode->i_mtime = ii->mtime; + inode->i_ctime = ii->ctime; + + if (S_ISLNK(inode->i_mode)) { + if (is_sb_proc(sb)) { + qtfs_info("inode link ops set to qtfs_proc_sym_ops."); + inode->i_op = &qtfs_proc_sym_ops; + } else { + inode->i_op = &qtfs_symlink_inode_ops; + } + } else { + if (is_sb_proc(sb)) { + inode->i_op = &qtfs_proc_inode_ops; + } else { + inode->i_op = &qtfs_inode_ops; + } + } + inode->i_mapping->a_ops = &qtfs_aops; + + if (S_ISDIR(ii->mode)) { + inode->i_fop = &qtfs_dir_ops; + } else if (S_ISREG(ii->mode)) { + inode->i_fop = &qtfs_file_ops; + } else if (S_ISFIFO(ii->mode)) { + inode->i_fop = &qtfsfifo_ops; + } else { + inode->i_fop = &qtfs_file_ops; + } + qtfs_inode_priv_alloc(inode); + return; +} + +struct inode *qtfs_iget(struct super_block *sb, struct inode_info *ii) +{ + struct inode *inode; + + inode = new_inode(sb); + if (!inode) + return NULL; + qtfs_init_inode(sb, inode, ii); + return inode; +} + +struct dentry *qtfs_lookup(struct inode *parent_inode, struct dentry *child_dentry, unsigned int flags) +{ + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_lookup *req; + struct qtrsp_lookup *rsp; + struct inode *inode; + struct dentry *d = NULL; + int ret; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var"); + return NULL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + ret = qtfs_fullname(req->fullname, child_dentry, sizeof(req->fullname)); + if (ret < 0) { + qtfs_err("qtfs lookup get fullname failed, too many path layers, <%s>!", req->fullname); + goto err_end; + } + rsp = qtfs_remote_run(pvar, QTFS_REQ_LOOKUP, strlen(req->fullname)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return (void *)rsp; + } + if (rsp->ret != QTFS_OK) { + qtfs_info("qtfs fs lookup failed, path:<%s> not exist at peer.\n", req->fullname); + d = ERR_PTR(rsp->errno); + qtfs_conn_put_param(pvar); + return d; + } + inode = qtfs_iget(parent_inode->i_sb, &(rsp->inode_info)); + if (inode == NULL) + goto err_end; + d = d_splice_alias(inode, child_dentry); + qtfs_debug("qtfs lookup fullname:%s mode:%o(rsp:%o), ino:%lu(rsp:%lu).", + req->fullname, inode->i_mode, rsp->inode_info.mode, inode->i_ino, rsp->inode_info.i_ino); + if (d) { + if (d->d_inode && S_ISDIR(d->d_inode->i_mode)) + d->d_time = jiffies; + } + + qtfs_conn_put_param(pvar); + return d; + +err_end: + qtfs_conn_put_param(pvar); + return NULL; +} +int qtfs_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_rmdir *req; + struct qtrsp_rmdir *rsp; + int ret; + struct inode *inode = d_inode(dentry); + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return -EINVAL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, dentry, sizeof(req->path)); + + rsp = qtfs_remote_run(pvar, QTFS_REQ_RMDIR, QTFS_SEND_SIZE(struct qtreq_rmdir, req->path)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(rsp); + } + + if (rsp->ret == QTFS_ERR) { + qtfs_err("qtfs rmdir <%s> failed, errno:%d.\n", req->path, rsp->errno); + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; + } + qtfs_info("qtfs rmdir success:<%s>.\n", req->path); + qtfs_conn_put_param(pvar); + if (inode->i_nlink > 0) + drop_nlink(inode); + d_invalidate(dentry); + return 0; +} + +int qtfs_unlink(struct inode *dir, struct dentry *dentry) +{ + struct qtreq_unlink *req; + struct qtrsp_unlink *rsp; + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + int ret; + struct inode *inode = d_inode(dentry); + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return -EINVAL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, dentry, sizeof(req->path)); + qtfs_info("qtfs unlink %s.\n", req->path); + + rsp = qtfs_remote_run(pvar, QTFS_REQ_UNLINK, QTFS_SEND_SIZE(struct qtreq_unlink, req->path)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(rsp); + } + if (rsp->errno < 0) { + qtfs_err("qtfs unlink %s failed, errno:%d\n", req->path, rsp->errno); + } else { + qtfs_info("qtfs unlink %s success\n", req->path); + inode->i_ctime = dir->i_ctime; + inode_dec_link_count(inode); + } + ret = rsp->errno; + qtfs_conn_put_param(pvar); + if (inode->i_nlink > 0) + drop_nlink(inode); + d_invalidate(dentry); + return ret; +} + +int qtfs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry) +{ + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_link *req; + struct qtrsp_link *rsp; + int error; + struct inode *inode = d_inode(old_dentry); + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return -EINVAL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, old_dentry, sizeof(req->path)); + req->d.oldlen = strlen(req->path) + 1; + QTFS_FULLNAME(req->path + req->d.oldlen, new_dentry, sizeof(req->path) - req->d.oldlen); + req->d.newlen = strlen(req->path + req->d.oldlen) + 1; + rsp = qtfs_remote_run(pvar, QTFS_REQ_LINK, sizeof(struct qtreq_link) - sizeof(req->path) + req->d.newlen + req->d.oldlen); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(rsp); + } + if (rsp->ret == QTFS_ERR) { + qtfs_err("qtfs link failed %d\n", rsp->errno); + error = rsp->errno; + goto err_end; + } + inode->i_ctime = current_time(inode); + inode_inc_link_count(inode); + ihold(inode); + d_instantiate(new_dentry, inode); + qtfs_info("qtfs link success, old:%s new:%s", req->path, req->path + req->d.oldlen); + qtfs_conn_put_param(pvar); + return 0; + +err_end: + qtfs_conn_put_param(pvar); + return error; +} + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) +int qtfs_symlink(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, const char *symname) +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) +int qtfs_symlink(struct user_namespace *mnt_userns, struct inode *dir, struct dentry *dentry, const char *symname) +#else +int qtfs_symlink(struct inode *dir, struct dentry *dentry, const char *symname) +#endif +{ + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_symlink *req; + struct qtrsp_symlink *rsp; + struct inode *inode; + int error; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return -EINVAL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, dentry, sizeof(req->path)); + req->d.newlen = strlen(req->path) + 1; + if (req->d.newlen + strlen(symname) + 1 > sizeof(req->path)) { + qtfs_conn_put_param(pvar); + qtfs_err("qtfs symlink path name too long\n"); + return -EINVAL; + } + strlcpy(&req->path[req->d.newlen], symname, sizeof(req->path) - req->d.newlen - 1); + + req->d.oldlen = strlen(&req->path[req->d.newlen]) + 1; + rsp = qtfs_remote_run(pvar, QTFS_REQ_SYMLINK, sizeof(struct qtreq_symlink) - sizeof(req->path) + req->d.newlen + req->d.oldlen); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(rsp); + } + if (rsp->ret == QTFS_ERR) { + qtfs_err("qtfs symlink failed %d\n", rsp->errno); + error = rsp->errno; + goto err_end; + } + inode = qtfs_iget(dentry->d_sb, &(rsp->inode_info)); + error = qtfs_new_entry(inode, dentry); + qtfs_info("qtfs symlink success, path:%s symname:%s", req->path, symname); + qtfs_conn_put_param(pvar); + return error; + +err_end: + qtfs_conn_put_param(pvar); + return error; +} + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) +int qtfs_getattr(struct mnt_idmap *idmap, const struct path *path, struct kstat *stat, u32 req_mask, unsigned int flags) +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) +int qtfs_getattr(struct user_namespace *mnt_userns, const struct path *path, struct kstat *stat, u32 req_mask, unsigned int flags) +#else +int qtfs_getattr(const struct path *path, struct kstat *stat, u32 req_mask, unsigned int flags) +#endif +{ + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_getattr *req; + struct qtrsp_getattr *rsp; + char *mnt_path = NULL; + struct inode *inode = path->dentry->d_inode; + int ret; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return -EINVAL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, path->dentry, sizeof(req->path)); + req->request_mask = req_mask; + req->query_flags = flags; + mnt_path = qtfs_mountpoint_path_init(path->dentry, (struct path*)path, req->path); + if (IS_ERR(mnt_path)) { + qtfs_conn_put_param(pvar); + return PTR_ERR(mnt_path); + } + rsp = qtfs_remote_run(pvar, QTFS_REQ_GETATTR, QTFS_SEND_SIZE(struct qtreq_getattr, req->path)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(rsp); + } + if (rsp->ret) { + qtfs_err("qtfs getattr <%s> failed.errno: %d %s\n", req->path, rsp->errno, + (rsp->errno != -ENOENT) ? "." : "file not exist"); + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; + } + *stat = rsp->stat; + if (path->dentry && path->dentry->d_inode && S_ISDIR(path->dentry->d_inode->i_mode)) + path->dentry->d_time = jiffies; + qtfs_debug("qtfs getattr success:<%s> blksiz:%u size:%lld mode:%o ino:%llu pathino:%lu. %s\n", req->path, rsp->stat.blksize, + rsp->stat.size, rsp->stat.mode, rsp->stat.ino, inode->i_ino, rsp->stat.ino != inode->i_ino ? "delete current inode" : ""); + if (inode->i_ino != rsp->stat.ino || inode->i_mode != rsp->stat.mode) { + if (inode->i_nlink > 0){ + drop_nlink(inode); + } + d_invalidate(path->dentry); + } + qtfs_conn_put_param(pvar); + return 0; +} + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) +int qtfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr) +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) +int qtfs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, struct iattr *attr) +#else +int qtfs_setattr(struct dentry *dentry, struct iattr *attr) +#endif +{ + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_setattr *req; + struct qtrsp_setattr *rsp; + int ret; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return -EINVAL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, dentry, sizeof(req->path)); + req->attr = *attr; + req->attr.ia_file = NULL; + qtfs_info("iattr iavalid:%u mode:0x%o size:%lld\n", + req->attr.ia_valid, req->attr.ia_mode, req->attr.ia_size); + rsp = qtfs_remote_run(pvar, QTFS_REQ_SETATTR, QTFS_SEND_SIZE(struct qtreq_setattr, req->path)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(rsp); + } + if (rsp->ret == QTFS_ERR) { + qtfs_err("qtfs setattr <%s> failed. %d\n", req->path, rsp->errno); + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; + } + qtfs_info("qtfs setattr <%s> success.\n", req->path); + qtfs_conn_put_param(pvar); + return 0; +} +const char *qtfs_getlink(struct dentry *dentry, + struct inode *inode, struct delayed_call *done) +{ + struct qtfs_conn_var_s *pvar = NULL; + struct qtreq_getlink *req; + struct qtrsp_getlink *rsp; + size_t len = 0; + struct qtfs_fs_info *fsinfo = qtfs_priv_byinode(inode); + char *link = NULL; + + link = READ_ONCE(inode->i_link); + if (link) { + qtfs_info("qtfs get link cache.\n"); + return link; + } + + if (dentry == NULL) { + return ERR_PTR(-ECHILD); + } + pvar = qtfs_conn_get_param(); + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return ERR_PTR(-EINVAL); + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + if (qtfs_fullname(req->path, dentry, sizeof(req->path)) < 0) { + qtfs_err("qtfs fullname failed\n"); + qtfs_conn_put_param(pvar); + return ERR_PTR(-EINVAL); + } + rsp = qtfs_remote_run(pvar, QTFS_REQ_GETLINK, QTFS_SEND_SIZE(struct qtreq_getlink, req->path)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return (void *)rsp; + } + if (rsp->ret == QTFS_ERR || strnlen(rsp->path, sizeof(rsp->path)) >= sizeof(rsp->path)) { + qtfs_err("qtfs getlink <%s> failed. %d\n", req->path, rsp->errno); + qtfs_conn_put_param(pvar); + return ERR_PTR(-ENOENT); + } + if (fsinfo->mnt_path) + len = strlen(fsinfo->mnt_path) + strlen(rsp->path) + 1; + else + len = strlen(rsp->path) + 1; + if (len > MAX_PATH_LEN || len == 0) { + qtfs_err("qtfs getlink failed. path name too long:%s - %s\n", fsinfo->mnt_path, rsp->path); + qtfs_conn_put_param(pvar); + return ERR_PTR(-EINVAL); + } + link = kmalloc(len, GFP_KERNEL); + if (!link) { + qtfs_conn_put_param(pvar); + return ERR_PTR(-ENOMEM); + } + memset(link, 0, len); + if (rsp->path[0] == '/' && fsinfo->mnt_path) + strcat(link, fsinfo->mnt_path); + strcat(link, rsp->path); + qtfs_info("get link success <%s>\n", link); + + set_delayed_call(done, kfree_link, link); + qtfs_conn_put_param(pvar); + return link; +} + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) +int qtfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) +int qtfs_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) + +#else +int qtfs_rename(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) +#endif +{ + struct qtreq_rename *req; + struct qtrsp_rename *rsp; + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + int ret; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return -EINVAL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + req->d.oldlen = qtfs_fullname(req->path, old_dentry, sizeof(req->path)); + if (req->d.oldlen < 0) { + qtfs_err("qtfs fullname failed\n"); + qtfs_conn_put_param(pvar); + return -EINVAL; + } + req->d.oldlen += 1; + req->d.newlen = qtfs_fullname(&req->path[req->d.oldlen], new_dentry, sizeof(req->path) - req->d.oldlen); + if (req->d.newlen < 0) { + qtfs_err("qtfs fullname failed\n"); + qtfs_conn_put_param(pvar); + return -EINVAL; + } + req->d.newlen += 1; + req->d.flags = flags; + + rsp = qtfs_remote_run(pvar, QTFS_REQ_RENAME, sizeof(struct qtreq_rename) - sizeof(req->path) + req->d.oldlen + req->d.newlen); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(rsp); + } + + if (rsp->ret == QTFS_ERR) { + qtfs_err("qtfs rename failed,errno:%d\n", rsp->errno); + } else { + qtfs_info("qtfs rename success, oldname:%s newname:%s flags:%x\n", req->path, &req->path[req->d.oldlen], flags); + } + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; +} + +static struct inode_operations qtfs_inode_ops = { + .create = qtfs_create, + .lookup = qtfs_lookup, + .mkdir = qtfs_mkdir, + .rmdir = qtfs_rmdir, + .unlink = qtfs_unlink, + .symlink = qtfs_symlink, + .link = qtfs_link, + .mknod = qtfs_mknod, + .getattr = qtfs_getattr, + .setattr = qtfs_setattr, + .rename = qtfs_rename, + .listxattr = qtfs_xattr_list, +}; + +static struct inode_operations qtfs_symlink_inode_ops = { + .get_link = qtfs_getlink, + .getattr = qtfs_getattr, + .setattr = qtfs_setattr, + .listxattr = qtfs_xattr_list, +}; + +const struct xattr_handler *qtfs_xattr_handlers[] = { + &qtfs_xattr_user_handler, + &qtfs_xattr_trusted_handler, + &qtfs_xattr_security_handler, +#ifndef KVER_4_19 + &qtfs_xattr_hurd_handler, +#endif + NULL +}; + +int qtfs_dentry_revalidate(struct dentry *dentry, unsigned int flags) +{ + struct qtfs_conn_var_s *pvar = NULL; + struct qtreq_getattr *req; + struct qtrsp_getattr *rsp; + struct inode *inode = NULL; + if (dentry && dentry->d_inode) { + if (jiffies_to_msecs(jiffies - dentry->d_time) < 2000) + return 1; + pvar = qtfs_conn_get_param(); + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return 0; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + qtfs_fullname(req->path, dentry, PATH_MAX); + req->request_mask = STATX_BASIC_STATS; + req->query_flags = 0; + + rsp = qtfs_remote_run(pvar, QTFS_REQ_GETATTR, QTFS_SEND_SIZE(struct qtreq_getattr, req->path)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return 0; + } + if (rsp->ret) { + qtfs_conn_put_param(pvar); + return 0; + } + + inode = dentry->d_inode; + if (inode == NULL) { + qtfs_conn_put_param(pvar); + return 0; + } + if (inode->i_ino != rsp->stat.ino || inode->i_mode != rsp->stat.mode) { + if (inode->i_nlink > 0) + drop_nlink(inode); + qtfs_conn_put_param(pvar); + return 0; + } + qtfs_conn_put_param(pvar); + dentry->d_time = jiffies; + } + return 1; +} + +const struct dentry_operations qtfs_dentry_ops = { + .d_revalidate = qtfs_dentry_revalidate, +}; + +static int qtfs_fill_super(struct super_block *sb, void *priv_data, int silent) +{ + struct inode *root_inode; + int mode = S_IFDIR; + int err; + struct qtfs_fs_info *priv = (struct qtfs_fs_info *)priv_data; + + root_inode = new_inode(sb); + root_inode->i_ino = 1; +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) + inode_init_owner(&nop_mnt_idmap, root_inode, NULL, mode); +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) + inode_init_owner(&init_user_ns, root_inode, NULL, mode); +#else + inode_init_owner(root_inode, NULL, mode); +#endif + root_inode->i_sb = sb; + if (priv->type == QTFS_PROC) { + qtfs_info("qtfs type: proc\n"); + root_inode->i_op = &qtfs_proc_inode_ops; + } else { + qtfs_info("qtfs type: normal\n"); + root_inode->i_op = &qtfs_inode_ops; + } + root_inode->i_fop = &qtfs_dir_ops; + root_inode->i_atime = root_inode->i_mtime = root_inode->i_ctime = CURRENT_TIME(root_inode); + + sb->s_xattr = qtfs_xattr_handlers; + err = super_setup_bdi(sb); + if (err) { + qtfs_err("qtfs fill super bdi setup err:%d.\n", err); + } + sb->s_fs_info = priv; + sb->s_op = &qtfs_ops; + sb->s_time_gran = 1; + sb->s_d_op = &qtfs_dentry_ops; + + sb->s_root = d_make_root(root_inode); + return 0; +} + +struct dentry *qtfs_fs_mount(struct file_system_type *fs_type, + int flags, const char *dev_name, void *data) +{ + struct qtreq_mount *req = NULL; + struct qtrsp_mount *rsp = NULL; + struct dentry *ret; + struct qtfs_fs_info *priv = NULL; + int errno; + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return ERR_PTR(-ENXIO); + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + strlcpy(req->path, dev_name, PATH_MAX); + rsp = qtfs_remote_run(pvar, QTFS_REQ_MOUNT, strlen(dev_name)); + if (IS_ERR_OR_NULL(rsp) || rsp->ret != QTFS_OK) { + errno = IS_ERR_OR_NULL(rsp) ? -EFAULT : rsp->errno; + qtfs_err("qtfs fs mount failed, path:<%s> errno:%d.\n", dev_name, errno); + qtfs_conn_put_param(pvar); + return (IS_ERR_VALUE((long)errno)) ? ERR_PTR(errno) : ERR_PTR(-EFAULT); + } + + priv = (struct qtfs_fs_info *)kmalloc(sizeof(struct qtfs_fs_info), GFP_KERNEL); + if (IS_ERR_OR_NULL(priv)) { + qtfs_err("qtfs priv kmalloc failed:%ld\n", QTFS_PTR_ERR(priv)); + qtfs_conn_put_param(pvar); + return ERR_PTR(-ENOMEM); + } + + memset(priv, 0, sizeof(struct qtfs_fs_info)); + priv->type = qtfs_get_type((char *)data); + strlcpy(priv->peer_path, dev_name, NAME_MAX); + priv->mnt_path = NULL; + + ret = mount_nodev(fs_type, flags, (void *)priv, qtfs_fill_super); + if (IS_ERR_OR_NULL(ret)) { + qtfs_err("mount qtfs error.\n"); + } else { + qtfs_info("mount qtfs success dev name:%s.\n", dev_name); + } + + qtfs_conn_put_param(pvar); + return ret; +} + +void qtfs_kill_sb(struct super_block *sb) +{ + struct qtfs_fs_info *fsinfo = sb->s_fs_info; + if (fsinfo->mnt_path) { + kfree(fsinfo->mnt_path); + fsinfo->mnt_path = NULL; + } + kfree(fsinfo); + sb->s_fs_info = NULL; + qtfs_info("qtfs superblock deleted.\n"); + kill_anon_super(sb); +} + diff --git a/qtfs/qtfs/syscall.c b/qtfs/qtfs/syscall.c new file mode 100644 index 0000000..f272b0d --- /dev/null +++ b/qtfs/qtfs/syscall.c @@ -0,0 +1,434 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "conn.h" +#include "qtfs-mod.h" +#include "symbol_wrapper.h" + +static long qtfs_remote_mount(char __user *dev_name, char __user *dir_name, char __user *type, + unsigned long flags, void __user *data); +static int qtfs_remote_umount(char __user *name, int flags); + +#ifdef BEFORE_KVER_5_6 +static inline int ep_op_has_event(int op) +{ + return op != EPOLL_CTL_DEL; +} +#endif + +static char *qtfs_copy_mount_string(const void __user *data) +{ + return data ? strndup_user(data, PATH_MAX) : NULL; +} + +static inline int qtfs_fstype_judgment(char __user *dir) +{ + struct path path; + int ret; + + ret = user_path_at(AT_FDCWD, dir, LOOKUP_FOLLOW, &path); + if (ret) + return 0; + + if (path.mnt && path.mnt->mnt_sb && + path.mnt->mnt_sb->s_type && path.mnt->mnt_sb->s_type->name && + strcmp(path.mnt->mnt_sb->s_type->name, QTFS_FSTYPE_NAME) == 0) { + qtfs_info("qtfs fstype judge <%s> is qtfs.\n", path.dentry->d_iname); + path_put(&path); + return 1; + } + path_put(&path); + + return 0; +} + +/* if this dir is root node of qtfs */ +static inline int qtfs_root_judgment(char __user *dir) +{ + struct dentry *dentry; + struct path path; + int ret = 0; + + ret = user_path_at(AT_FDCWD, dir, LOOKUP_FOLLOW, &path); + if (ret) + return 0; + + dentry = path.dentry; + if (dentry->d_parent == dentry) + ret = 1; + path_put(&path); + + return ret; +} + +static void do_epoll_ctl_remote(int op, struct epoll_event __user *event, struct file *file) +{ + struct qtreq_epollctl *req; + struct qtrsp_epollctl *rsp; + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct private_data *priv = file->private_data; + struct epoll_event tmp; + + if (pvar == NULL) { + qtfs_err("qtfs do epoll ctl remote get pvar failed."); + return; + } + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + req->fd = priv->fd; + req->op = op; + if (ep_op_has_event(op) && copy_from_user(&tmp, event, sizeof(struct epoll_event))) { + qtfs_err("qtfs do epoll ctl remote copy from user failed."); + qtfs_conn_put_param(pvar); + return; + } + req->event.events = tmp.events; + req->event.data = (__u64)file; + rsp = qtfs_remote_run(pvar, QTFS_REQ_EPOLL_CTL, sizeof(struct qtreq_epollctl)); + if (IS_ERR_OR_NULL(rsp) || rsp->ret == QTFS_ERR) { + qtfs_err("qtfs do epoll ctl remote failed."); + qtfs_conn_put_param(pvar); + qtinfo_cntinc(QTINF_EPOLL_FDERR); + return; + } + if (op == EPOLL_CTL_ADD) { + qtinfo_cntinc(QTINF_EPOLL_ADDFDS); + } else { + qtinfo_cntinc(QTINF_EPOLL_DELFDS); + } + qtfs_info("qtfs do epoll ctl remote success, fd:%d.", req->fd); + qtfs_conn_put_param(pvar); + return; +} + +int qtfs_epoll_ctl_remote(int op, int fd, struct epoll_event __user * event) +{ + struct fd f; + struct file *file; + struct private_data *priv; + int ret = 0; + f = fdget(fd); + if (!f.file) { + return -1; + } + file = f.file; + if (strcmp(file->f_path.mnt->mnt_sb->s_type->name, QTFS_FSTYPE_NAME) != 0) { + ret = 0; + goto end; + } + if (!qtfs_support_epoll(file->f_inode->i_mode)) { + char *fullname = (char *)kmalloc(MAX_PATH_LEN, GFP_KERNEL); + if (!fullname) { + ret = -1; + goto end; + } + memset(fullname, 0, MAX_PATH_LEN); + if (qtfs_fullname(fullname, file->f_path.dentry, MAX_PATH_LEN) < 0) { + qtfs_err("qtfs fullname failed\n"); + kfree(fullname); + ret = -1; + goto end; + } + qtfs_info("qtfs remote epoll not support file:%s mode:%o.", fullname, file->f_inode->i_mode); + kfree(fullname); + ret = -1; + goto end; + } + + priv = file->private_data; + if (priv == NULL) { + qtfs_err("epoll ctl remote failed, private data invalid."); + ret = -1; + goto end; + } + + qtfs_info("qtfs qtfs remote epoll file:%s mode:%x file can poll.", + file->f_path.dentry->d_iname, file->f_inode->i_mode); + do_epoll_ctl_remote(op, event, file); + +end: + fdput(f); + return ret; +} + +__SYSCALL_DEFINEx(4, _qtfs_epoll_ctl, int, epfd, int, op, int, fd, + struct epoll_event __user *, event) +{ + int ret = -1; + + ret = qtfs_epoll_ctl_remote(op, fd, event); + if (!ret) { + return qtfs_syscall_epoll_ctl(epfd, op, fd, event); + } else { + return -1; + } +} + +__SYSCALL_DEFINEx(5, _qtfs_mount, char __user *, dev_name, char __user *, dir_name, + char __user *, type, unsigned long, flags, void __user *, data) +{ + int ret; + char *kernel_type; + char *kernel_dev; + void *options = NULL; + + // if both dev_name and dir_name are qtfs, it is a remote mount operator. + kernel_type = qtfs_copy_mount_string(type); + ret = PTR_ERR(kernel_type); + if (IS_ERR(kernel_type)) + goto out_type; + + kernel_dev = qtfs_copy_mount_string(dev_name); + ret = PTR_ERR(kernel_dev); + if (IS_ERR(kernel_dev)) + goto out_dev; + + options = qtfs_copy_mount_string(data); + ret = PTR_ERR(options); + if (IS_ERR(options)) + goto out_data; + + // if both dev_name and dir_name are qtfs, it is a remote mount operator, + if (qtfs_fstype_judgment(dir_name) == 1) { + ret = qtfs_remote_mount(kernel_dev, dir_name, kernel_type, flags, options); + goto remote_mount; + } + + ret = qtfs_syscall_mount(dev_name, dir_name, type, flags, data); + +remote_mount: + kfree(options); +out_data: + kfree(kernel_dev); +out_dev: + kfree(kernel_type); +out_type: + return ret; +} + +__SYSCALL_DEFINEx(2, _qtfs_umount, char __user *, name, int, flags) +{ + // basic validate checks done first + if (flags & ~(MNT_FORCE | MNT_DETACH | MNT_EXPIRE | UMOUNT_NOFOLLOW)) + return -EINVAL; + + /* if umount path is qtfs and not qtfs root, then do remote umount */ + if (qtfs_fstype_judgment(name) && !qtfs_root_judgment(name)) { + return qtfs_remote_umount(name, flags); + } + + return qtfs_syscall_umount(name, flags); +} + +int qtfs_dir_to_qtdir(char *dir, char *qtdir, size_t len) +{ + int ret = 0; + struct path path; + + if (strlen(dir) + 1 > len) { + strlcpy(qtdir, dir, len); + return -EINVAL; + } + ret = kern_path(dir, 0, &path); + if (ret) { + strlcpy(qtdir, dir, len); + return 0; + } + if (strcmp(path.mnt->mnt_sb->s_type->name, QTFS_FSTYPE_NAME)) { + strlcpy(qtdir, dir, len); + } else { + ret = qtfs_fullname(qtdir, path.dentry, len); + } + path_put(&path); + return ret; +} + +static size_t qtfs_strlen(const char *s) +{ + if (s == NULL) + return 0; + return strlen(s); +} + +static long qtfs_remote_mount(char *dev_name, char __user *dir_name, char *type, + unsigned long flags, void *data) +{ + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_sysmount *req; + struct qtrsp_sysmount *rsp = NULL; + char *kernel_dir; + int ret; + size_t totallen; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return -EINVAL; + } + kernel_dir = qtfs_copy_mount_string(dir_name); + if (IS_ERR_OR_NULL(kernel_dir)) { + qtfs_conn_put_param(pvar); + return -EINVAL; + } + totallen = qtfs_strlen(dev_name) + qtfs_strlen(kernel_dir) + qtfs_strlen(type) + qtfs_strlen(data) + 4; + if (totallen > sizeof(req->buf)) { + qtfs_err("qtfs remote mount devname:%s, dir_name:%s failed, options too long.\n", dev_name, kernel_dir); + kfree(kernel_dir); + qtfs_conn_put_param(pvar); + return -EINVAL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + if (dev_name != NULL) { + qtfs_dir_to_qtdir(dev_name, req->buf, sizeof(req->buf)); + req->d.dev_len = strlen(dev_name) + 1; + } else { + req->d.dev_len = 0; + } + + qtfs_dir_to_qtdir(kernel_dir, &req->buf[req->d.dev_len], sizeof(req->buf) - req->d.dev_len); + req->d.dir_len = strlen(&req->buf[req->d.dev_len]) + 1; + if (type != NULL) { + strlcpy(&req->buf[req->d.dev_len + req->d.dir_len], type, strlen(type) + 1); + req->d.type_len = strlen(type) + 1; + } else { + req->d.type_len = 0; + } + + if (data != NULL) { + req->d.data_len = strlen(data) + 1; + strlcpy(&req->buf[req->d.dev_len + req->d.dir_len + req->d.type_len], data, strlen(data) + 1); + } else { + req->d.data_len = 0; + } + req->d.flags = flags; + + rsp = qtfs_remote_run(pvar, QTFS_REQ_SYSMOUNT, sizeof(struct qtreq_sysmount) - sizeof(req->buf) + totallen); + if (IS_ERR_OR_NULL(rsp)) { + kfree(kernel_dir); + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(rsp); + } + if (rsp->errno < 0) { + qtfs_err("qtfs remote mount failed, devname:%s dir_name:%s type:%s, data:%s, flags(0x%lx), errno:%d\n", + dev_name, kernel_dir, type, (char *)data, flags, rsp->errno); + } else { + qtfs_info("qtfs remote mount success devname:%s dir_name:%s type:%s, data:%s, flags(0x%lx)\n", + dev_name, kernel_dir, type, (char *)data, flags); + } + + kfree(kernel_dir); + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; +} + +static int qtfs_remote_umount(char __user *name, int flags) +{ + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_sysumount *req; + struct qtrsp_sysumount *rsp; + char *kernel_name; + int ret; + + if (pvar == NULL) { + qtfs_err("qtfs remote umount get pvar failed."); + return -EINVAL; + } + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + kernel_name = qtfs_copy_mount_string(name); + if (IS_ERR_OR_NULL(kernel_name)) { + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(kernel_name); + } + req->flags = flags; + qtfs_dir_to_qtdir(kernel_name, req->buf, sizeof(req->buf)); + qtfs_info("qtfs remote umount string:%s reqbuf:%s", (kernel_name == NULL) ? "INVALID":kernel_name, req->buf); + + rsp = qtfs_remote_run(pvar, QTFS_REQ_SYSUMOUNT, sizeof(struct qtreq_sysumount) - sizeof(req->buf) + strlen(req->buf)); + if (IS_ERR_OR_NULL(rsp)) { + kfree(kernel_name); + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(rsp); + } + if (rsp->errno) + qtfs_err("qtfs remote umount failed, errno:%d\n", rsp->errno); + + kfree(kernel_name); + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; +} + +static atomic_t replace_available = ATOMIC_INIT(1); + +int qtfs_syscall_init(void) +{ + if (!atomic_dec_and_test(&replace_available)) { + atomic_inc(&replace_available); + return -EBUSY; + } + + symbols_origin[SYMBOL_SYSCALL_MOUNT] = qtfs_kern_syms.sys_call_table[__NR_mount]; + symbols_origin[SYMBOL_SYSCALL_UMOUNT] = qtfs_kern_syms.sys_call_table[__NR_umount2]; + symbols_origin[SYMBOL_SYSCALL_EPOLL_CTL] = qtfs_kern_syms.sys_call_table[__NR_epoll_ctl]; +#ifdef __x86_64__ + make_rw((unsigned long)qtfs_kern_syms.sys_call_table); + qtfs_kern_syms.sys_call_table[__NR_mount] = (unsigned long *)__x64_sys_qtfs_mount; + qtfs_kern_syms.sys_call_table[__NR_umount2] = (unsigned long *)__x64_sys_qtfs_umount; + qtfs_kern_syms.sys_call_table[__NR_epoll_ctl] = (unsigned long *)__x64_sys_qtfs_epoll_ctl; + make_ro((unsigned long)qtfs_kern_syms.sys_call_table); +#endif +#ifdef __aarch64__ + // disable write protection + update_mapping_prot(__pa_symbol(start_rodata), (unsigned long)start_rodata, section_size, PAGE_KERNEL); + qtfs_kern_syms.sys_call_table[__NR_mount] = (unsigned long *)__arm64_sys_qtfs_mount; + qtfs_kern_syms.sys_call_table[__NR_umount2] = (unsigned long *)__arm64_sys_qtfs_umount; + qtfs_kern_syms.sys_call_table[__NR_epoll_ctl] = (unsigned long *)__arm64_sys_qtfs_epoll_ctl; + // enable write protection + update_mapping_prot(__pa_symbol(start_rodata), (unsigned long)start_rodata, section_size, PAGE_KERNEL_RO); +#endif + qtfs_debug("qtfs use qtfs_mount instead of mount and umount\n"); + qtfs_debug("qtfs use qtfs_epoll_ctl instead of epoll_ctl\n"); + return 0; +} + +int qtfs_syscall_fini(void) +{ +#ifdef __x86_64__ + make_rw((unsigned long)qtfs_kern_syms.sys_call_table); + qtfs_kern_syms.sys_call_table[__NR_mount] = (unsigned long *)symbols_origin[SYMBOL_SYSCALL_MOUNT]; + qtfs_kern_syms.sys_call_table[__NR_umount2] = (unsigned long *)symbols_origin[SYMBOL_SYSCALL_UMOUNT]; + qtfs_kern_syms.sys_call_table[__NR_epoll_ctl] = (unsigned long *)symbols_origin[SYMBOL_SYSCALL_EPOLL_CTL]; + /*set mkdir syscall to the original one */ + make_ro((unsigned long)qtfs_kern_syms.sys_call_table); +#endif +#ifdef __aarch64__ + // disable write protection + update_mapping_prot(__pa_symbol(start_rodata), (unsigned long)start_rodata, section_size, PAGE_KERNEL); + qtfs_kern_syms.sys_call_table[__NR_mount] = (unsigned long *)symbols_origin[SYMBOL_SYSCALL_MOUNT]; + qtfs_kern_syms.sys_call_table[__NR_umount2] = (unsigned long *)symbols_origin[SYMBOL_SYSCALL_UMOUNT]; + qtfs_kern_syms.sys_call_table[__NR_epoll_ctl] = (unsigned long *)symbols_origin[SYMBOL_SYSCALL_EPOLL_CTL]; + // enable write protection + update_mapping_prot(__pa_symbol(start_rodata), (unsigned long)start_rodata, section_size, PAGE_KERNEL_RO); +#endif + qtfs_info("qtfs mount umount and epoll_ctl resumed\n"); + atomic_inc(&replace_available); + return 0; +} diff --git a/qtfs/qtfs/syscall.h b/qtfs/qtfs/syscall.h new file mode 100644 index 0000000..4284863 --- /dev/null +++ b/qtfs/qtfs/syscall.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __QTFS_SYSCALL_H__ +#define __QTFS_SYSCALL_H__ + +extern int qtfs_syscall_init(void); +extern int qtfs_syscall_fini(void); + +#endif diff --git a/qtfs/qtfs/xattr.c b/qtfs/qtfs/xattr.c new file mode 100644 index 0000000..4d15daf --- /dev/null +++ b/qtfs/qtfs/xattr.c @@ -0,0 +1,318 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include + +#include "conn.h" +#include "qtfs-mod.h" +#include "req.h" +#include "log.h" + +ssize_t qtfs_xattr_list(struct dentry *dentry, char *buffer, size_t buffer_size) +{ + struct qtreq_xattrlist *req; + struct qtrsp_xattrlist *rsp; + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + ssize_t ret; + + if (!pvar) { + qtfs_err("qtfs_xattr_list Failed to get qtfs sock var"); + return 0; + } + + if (dentry == NULL) { + qtfs_err("qtfs_xattr_list dentry is NULL."); + qtfs_conn_put_param(pvar); + return 0; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + if (qtfs_fullname(req->path, dentry, sizeof(req->path)) < 0) { + qtfs_err("qtfs fullname failed"); + qtfs_conn_put_param(pvar); + return 0; + } + req->buffer_size = buffer_size; + rsp = qtfs_remote_run(pvar, QTFS_REQ_XATTRLIST, QTFS_SEND_SIZE(struct qtreq_xattrlist, req->path)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_err("qtfs_xattr_list remote run failed."); + qtfs_conn_put_param(pvar); + return 0; + } + + if (rsp->d.ret == QTFS_ERR) { + qtfs_err("qtfs_xattr_list failed with ret:%d.", rsp->d.ret); + ret = rsp->d.size; + qtfs_conn_put_param(pvar); + return ret; + } + ret = rsp->d.size; + if (buffer != NULL) { + ret = (rsp->d.size > buffer_size) ? buffer_size : rsp->d.size; + memcpy(buffer, rsp->name, ret); + } + qtfs_conn_put_param(pvar); + return ret; +} + +static int qtfs_xattr_set(const struct xattr_handler *handler, + struct dentry *dentry, struct inode *inode, + const char *name, const void *value, + size_t size, int flags); + +static int +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) +qtfs_xattr_user_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) +qtfs_xattr_user_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +#else +qtfs_xattr_user_set(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +#endif + +{ + return qtfs_xattr_set(handler, unused, inode, name, value, size, flags); +} + +static int +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) +qtfs_xattr_trusted_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) +qtfs_xattr_trusted_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +#else +qtfs_xattr_trusted_set(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +#endif +{ + return qtfs_xattr_set(handler, unused, inode, name, value, size, flags); +} + +static int +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) +qtfs_xattr_security_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) +qtfs_xattr_security_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +#else +qtfs_xattr_security_set(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +#endif +{ + return qtfs_xattr_set(handler, unused, inode, name, value, size, flags); +} + +static int qtfs_xattr_set(const struct xattr_handler *handler, + struct dentry *dentry, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + struct qtreq_xattrset *req; + struct qtrsp_xattrset *rsp; + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + int ret; + size_t totallen; + + if (!pvar) { + qtfs_err("failed to get qtfs sock var"); + return -ENOMEM; + } + if (dentry == NULL) { + qtfs_conn_put_param(pvar); + return -ENOENT; + } + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + if (qtfs_fullname(req->buf, dentry, sizeof(req->buf)) < 0) { + qtfs_err("xattr set get fullname failed."); + qtfs_conn_put_param(pvar); + return -EFAULT; + } + req->d.valuelen = size; + req->d.flags = flags; + req->d.pathlen = strlen(req->buf) + 1; + req->d.namelen = strlen(name) + strlen(handler->prefix) + 1; + qtfs_info("xattr set path:%s name:%s size:%lu", req->buf, name, size); + totallen = req->d.pathlen + req->d.namelen + size; + if (totallen >= sizeof(req->buf)) { + qtfs_err("xattr set namelen:%lu size:%lu is too long", req->d.namelen, size); + qtfs_conn_put_param(pvar); + return -EFAULT; + } + strlcpy(&req->buf[req->d.pathlen], handler->prefix, strlen(handler->prefix) + 1); + strcat(&req->buf[req->d.pathlen], name); + if (size > 0) { + memcpy(&req->buf[req->d.pathlen + req->d.namelen], value, size); + req->d.valuelen = size + 1; + } + + rsp = qtfs_remote_run(pvar, QTFS_REQ_XATTRSET, sizeof(struct qtreq_xattrset) - sizeof(req->buf) + req->d.pathlen + req->d.namelen + req->d.valuelen); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(rsp); + } + if (rsp->errno < 0) { + qtfs_err("xattr set failed file:%s name:%s", req->buf, name); + } else { + qtfs_info("xattr set successed file:%s name:%s", req->buf, name); + } + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; +} + +static int qtfs_xattr_get(const struct xattr_handler *handler, + struct dentry *dentry, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + struct qtreq_xattrget *req; + struct qtrsp_xattrget *rsp; + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + size_t leftlen = size; + char *buf = (char *)buffer; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var"); + return 0; + } + + if (dentry == NULL) { + qtfs_err("xattr get dentry is NULL."); + qtfs_conn_put_param(pvar); + return 0; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + if (qtfs_fullname(req->path, dentry, sizeof(req->path)) < 0) { + qtfs_err("qtfs fullname failed"); + qtfs_conn_put_param(pvar); + return 0; + } + + if (strlen(handler->prefix) + strlen(name) <= (sizeof(req->d.prefix_name) - 1)) { + strcpy(req->d.prefix_name, handler->prefix); + strcat(req->d.prefix_name, name); + } else { + qtfs_err("strcpy len too long"); + qtfs_conn_put_param(pvar); + return 0; + } + + rsp = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_RECV); + do { + req->d.pos = rsp->d.pos; + req->d.size = size; + rsp = qtfs_remote_run(pvar, QTFS_REQ_XATTRGET, QTFS_SEND_SIZE(struct qtreq_xattrget, req->path)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_err("rsp invalid, file:%s", req->path); + qtfs_conn_put_param(pvar); + return QTFS_PTR_ERR(rsp); + } + if (rsp->d.ret == QTFS_ERR || (size !=0 && (rsp->d.size > req->d.size || leftlen < rsp->d.size))) { + qtfs_err("ret:%d rsp size:%ld req size:%d leftlen:%lu", rsp->d.ret, rsp->d.size, + req->d.size, leftlen); + goto err_end; + } + if (size > 0 && rsp->d.size <= leftlen) { + memcpy(&buf[size - leftlen], rsp->buf, rsp->d.size); + } + leftlen -= rsp->d.size; + } while (leftlen > 0 && rsp->d.size > 0); + qtfs_info("qtfs getxattr success:<<%s>>", buf); + + qtfs_conn_put_param(pvar); + + return size - leftlen; + +err_end: + qtfs_conn_put_param(pvar); + return -ENODATA; +} + +const struct xattr_handler qtfs_xattr_user_handler = { + .prefix = XATTR_USER_PREFIX, + .get = qtfs_xattr_get, + .set = qtfs_xattr_user_set, +}; + +const struct xattr_handler qtfs_xattr_trusted_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .get = qtfs_xattr_get, + .set = qtfs_xattr_trusted_set, +}; + +const struct xattr_handler qtfs_xattr_security_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .get = qtfs_xattr_get, + .set = qtfs_xattr_security_set, +}; + +#ifndef KVER_4_19 +static int +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) +qtfs_xattr_hurd_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) +qtfs_xattr_hurd_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +#else +qtfs_xattr_hurd_set(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +#endif +{ + return qtfs_xattr_set(handler, unused, inode, name, value, size, flags); +} + +const struct xattr_handler qtfs_xattr_hurd_handler = { + .prefix = XATTR_HURD_PREFIX, + .get = qtfs_xattr_get, + .set = qtfs_xattr_hurd_set, +}; +#endif diff --git a/qtfs/qtfs_common/License b/qtfs/qtfs_common/License new file mode 100644 index 0000000..3912109 --- /dev/null +++ b/qtfs/qtfs_common/License @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/qtfs/qtfs_common/conn.c b/qtfs/qtfs_common/conn.c new file mode 100644 index 0000000..f05dc22 --- /dev/null +++ b/qtfs/qtfs_common/conn.c @@ -0,0 +1,951 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include + +#include "comm.h" +#include "conn.h" +#include "log.h" +#include "req.h" +#include "symbol_wrapper.h" +#include "uds_module.h" + +struct qtfs_pvar_ops_s *g_pvar_ops = NULL; +char qtfs_log_level[QTFS_LOGLEVEL_STRLEN] = {0}; +char qtfs_conn_type[20] = QTFS_CONN_SOCK_TYPE; +int log_level = LOG_ERROR; +static int qtfs_conn_max_conn = QTFS_MAX_THREADS; +struct qtinfo *qtfs_diag_info = NULL; +bool qtfs_epoll_mode = false; // true: support any mode; false: only support fifo + +static atomic_t g_qtfs_conn_num; +static struct list_head g_vld_lst; +static struct list_head g_busy_lst; +static struct llist_head g_lazy_put_llst; +static struct list_head g_fifo_lst; +static struct mutex g_param_mutex; +static struct mutex g_fifo_mutex; +int qtfs_mod_exiting = false; +struct qtfs_conn_var_s *qtfs_thread_var[QTFS_MAX_THREADS] = {NULL}; +struct qtfs_conn_var_s *qtfs_epoll_var = NULL; +#ifdef QTFS_SERVER +struct qtfs_server_userp_s *qtfs_userps = NULL; +#endif +#ifdef QTFS_CLIENT +struct kmem_cache *qtfs_fifo_pvar_cache; +#endif + +// try to connect remote uds server, only for unix domain socket +#define QTFS_UDS_PROXY_SUFFIX ".proxy" +int qtfs_uds_proxy_build(struct socket *sock, struct sockaddr_un *addr, int len) +{ + int ret; + struct uds_proxy_remote_conn_req req; + struct uds_proxy_remote_conn_rsp rsp; + struct sockaddr_un proxy = {.sun_family = AF_UNIX}; + struct socket *proxy_sock; + struct msghdr msgs; + struct msghdr msgr; + struct kvec vec; + + ret = sock_create_kern(&init_net, AF_UNIX, SOCK_STREAM, 0, &proxy_sock); + if (ret) { + qtfs_err("create proxy sock failed sun path:%s", addr->sun_path); + return -EFAULT; + } + memset(proxy.sun_path, 0, sizeof(proxy.sun_path)); + strlcpy(proxy.sun_path, UDS_BUILD_CONN_ADDR, strlen(UDS_BUILD_CONN_ADDR) + 1); + ret = sock->ops->connect(proxy_sock, (struct sockaddr *)&proxy, sizeof(proxy), SOCK_NONBLOCK); + if (ret) { + qtfs_err("connect to uds proxy failed"); + goto err_end; + } + memset(req.sun_path, 0, sizeof(req.sun_path)); + strlcpy(req.sun_path, addr->sun_path, sizeof(req.sun_path)); + memset(&msgs, 0, sizeof(struct msghdr)); + memset(&msgr, 0, sizeof(struct msghdr)); + req.type = sock->sk->sk_type; + vec.iov_base = &req; + vec.iov_len = sizeof(req); + ret = kernel_sendmsg(proxy_sock, &msgs, &vec, 1, vec.iov_len); + if (ret < 0) { + qtfs_err("send remote connect request failed:%d", ret); + goto err_end; + } + vec.iov_base = &rsp; + vec.iov_len = sizeof(rsp); + ret = kernel_recvmsg(proxy_sock, &msgr, &vec, 1, vec.iov_len, MSG_WAITALL); + if (ret <= 0) { + qtfs_err("recv remote connect response failed:%d", ret); + goto err_end; + } + if (rsp.ret == 0) { + goto err_end; + } + qtfs_info("try to build uds proxy successed, sun path:%s", addr->sun_path); + + sock_release(proxy_sock); + return 0; +err_end: + sock_release(proxy_sock); + return -ECONNREFUSED; +} + +static int qtfs_uds_remote_whitelist(const char *path) +{ + int i; + int ret = 1; + struct qtfs_wl_cap *cap; + read_lock(&g_qtfs_wl.rwlock); + cap = &g_qtfs_wl.cap[QTFS_WHITELIST_UDSCONNECT]; + for (i = 0; i < cap->nums; i++) { + if (strncmp(path, cap->item[i], strlen(cap->item[i])) == 0) { + if (strlen(path) > strlen(cap->item[i]) && path[strlen(cap->item[i])] != '/') { + continue; + } + ret = 0; + break; + } + } + read_unlock(&g_qtfs_wl.rwlock); + return ret; +} + +static inline int qtfs_uds_is_proxy(void) +{ +#define UDS_PROXYD_PRNAME "udsproxyd" + if (strlen(current->comm) == strlen(UDS_PROXYD_PRNAME) && + strncmp(current->comm, UDS_PROXYD_PRNAME, strlen(UDS_PROXYD_PRNAME)) == 0) + return 1; + return 0; +} + +static inline int qtfs_uds_is_rexec(void) +{ +#define REXEC_PRNAME "rexec" + if (strlen(current->comm) == strlen(REXEC_PRNAME) && + strncmp(current->comm, REXEC_PRNAME, strlen(REXEC_PRNAME)) == 0) + return 1; + return 0; +} + +int qtfs_uds_remote_connect_user(int fd, struct sockaddr __user *addr, int len) +{ + int sysret = -EINVAL; + int ret; + int err; + int un_headlen; + struct fd f; + struct socket *sock; + struct sockaddr_un addr_un; + struct sockaddr_un addr_proxy; + + if (qtfs_uds_is_rexec()) { + qtfs_info("Rexec process has no nessary to connect local server"); + goto try_conn_remote; + } + sysret = qtfs_syscall_connect(fd, addr, len); + // don't try remote uds connect if: 1.local connect successed; 2.this process is udsproxyd + if (sysret == 0 || qtfs_uds_is_proxy()) + return sysret; +try_conn_remote: + // len is passed from syscall input args directly. it's trustworthy + if (copy_from_user(&addr_un, addr, len)) { + qtfs_err("copy sockaddr failed."); + return sysret; + } + // don't try remote uds connect if sunpath not in whitelist + if (qtfs_uds_remote_whitelist(addr_un.sun_path) != 0) + return sysret; + if (addr_un.sun_family != AF_UNIX) + return sysret; + un_headlen = sizeof(struct sockaddr_un) - sizeof(addr_un.sun_path); + // 如果用户态给的参数长度不够，这里智能失败退出 + if (len < un_headlen || strlen(addr_un.sun_path) >= (len - un_headlen - strlen(QTFS_UDS_PROXY_SUFFIX))) { + qtfs_err("failed to try connect remote uds server, sun path:%s too long to add suffix:%s", + addr_un.sun_path, QTFS_UDS_PROXY_SUFFIX); + return sysret; + } + qtfs_info("uds connect failed:%d try to remote connect:%s.", sysret, addr_un.sun_path); + + f = fdget(fd); + if (f.file == NULL) { + return -EBADF; + } +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0)) + sock =sock_from_file(f.file); +#else + sock = sock_from_file(f.file, &err); +#endif + if (!sock) { + goto end; + } + // try to connect remote uds's proxy + ret = qtfs_uds_proxy_build(sock, &addr_un, len); + if (ret == 0) { + memcpy(&addr_proxy, &addr_un, sizeof(struct sockaddr_un)); + strlcat(addr_proxy.sun_path, QTFS_UDS_PROXY_SUFFIX, sizeof(addr_proxy.sun_path)); + if (copy_to_user(addr, &addr_proxy, (len > sizeof(struct sockaddr_un)) ? sizeof(struct sockaddr_un) : len)) { + qtfs_err("copy to addr failed sunpath:%s", addr_proxy.sun_path); + goto end; + } + sysret = qtfs_syscall_connect(fd, addr, len); + qtfs_info("try remote connect sunpath:%s ret:%d", addr_un.sun_path, sysret); + if (copy_to_user(addr, &addr_un, (len > sizeof(struct sockaddr_un)) ? sizeof(struct sockaddr_un) : len)) { + qtfs_err("resume addr failed"); + goto end; + } + } + +end: + fdput(f); + return sysret; +} + +int qtfs_conn_init(struct qtfs_conn_var_s *pvar) +{ + return pvar->conn_ops->conn_init(&pvar->conn_var, pvar->user_type); +} + +void qtfs_conn_fini(struct qtfs_conn_var_s *pvar) +{ + return pvar->conn_ops->conn_fini(&pvar->conn_var, pvar->user_type); +} + +#define MAGIC_U32(magic, n) ((magic >> (n * 8)) & 0xff) +static inline int qtfs_conn_sync_magic(struct qtfs_conn_var_s *pvar, bool block) +{ + u8 byte; + int ret; + if (pvar->magic_recv == 0) + return 0; + while (1) { + ret = pvar->conn_ops->conn_recv(&pvar->conn_var, &byte, 1, block); + if (ret <= 0) break; + if (byte != MAGIC_U32(pvar->magic_recv, 3)) continue; + ret = pvar->conn_ops->conn_recv(&pvar->conn_var, &byte, 1, block); + if (ret <= 0) break; + if (byte != MAGIC_U32(pvar->magic_recv, 2)) continue; + ret = pvar->conn_ops->conn_recv(&pvar->conn_var, &byte, 1, block); + if (ret <= 0) break; + if (byte != MAGIC_U32(pvar->magic_recv, 1)) continue; + ret = pvar->conn_ops->conn_recv(&pvar->conn_var, &byte, 1, block); + if (ret <= 0) break; + if (byte != MAGIC_U32(pvar->magic_recv, 0)) continue; + break; + } + if (ret < 0) { + if (ret != -EAGAIN) + qtfs_err("qtfs sync magic failed ret:%d byte:%u", ret, byte); + return ret; + } + return 0; +} + +int qtfs_conn_send(struct qtfs_conn_var_s *pvar) +{ + int ret = 0; + int iov_ret = 0; + if (pvar->vec_send.iov_len > pvar->send_max) + return -EMSGSIZE; + if (pvar->magic_send != 0) { + ret = pvar->conn_ops->conn_send(&pvar->conn_var, &pvar->magic_send, sizeof(pvar->magic_send)); + if (ret <= 0) { + qtfs_err("magic send failed, ret:%d", ret); + return ret; + } + } + pvar->send_valid = pvar->vec_send.iov_len; + ret = pvar->conn_ops->conn_send(&pvar->conn_var, pvar->vec_send.iov_base, pvar->vec_send.iov_len); + if (ret <= 0) + return ret; + if (pvar->iov_send) { + iov_ret = pvar->conn_ops->conn_send_iter(&pvar->conn_var, pvar->iov_send); + pvar->iov_send = NULL; // invalid it after use + if (iov_ret <= 0) + return iov_ret; + } + return ret + iov_ret; +} + +int do_qtfs_conn_recv(struct qtfs_conn_var_s *pvar, bool block) +{ + int ret = 0; + int headlen = 0; + struct qtreq *rsp = NULL; + struct kvec load; + unsigned long retrytimes = 0; + + headlen = pvar->conn_ops->conn_recv(&pvar->conn_var, pvar->vec_recv.iov_base, QTFS_MSG_HEAD_LEN, block); + + if (headlen <= 0) { + return headlen; + } + + load.iov_base = pvar->vec_recv.iov_base + QTFS_MSG_HEAD_LEN; + load.iov_len = pvar->vec_recv.iov_len - QTFS_MSG_HEAD_LEN; + rsp = pvar->vec_recv.iov_base; + + // only recv head + if (load.iov_len == 0) + goto end; + +retry: + ret = pvar->conn_ops->conn_recv(&pvar->conn_var, load.iov_base, + (rsp->len < load.iov_len) ? rsp->len : load.iov_len, true); + if (ret == -EAGAIN) + goto retry; + if (ret == -ERESTARTSYS) { +#ifdef QTFS_CLIENT + if (retrytimes == 0) { + qtinfo_cntinc(QTINF_RESTART_SYS); + qtinfo_recverrinc(rsp->type); + } +#endif + retrytimes++; + msleep(1); + goto retry; + } + if (ret < 0) { + qtfs_err("qtfs recv get invalidelen is :%d", ret); + return ret; + } + + if (ret > rsp->len) { + qtfs_crit("recv total:%d msg len:%lu\n", ret, rsp->len); + WARN_ON(1); + } +end: + return ret + headlen; +} + +int qtfs_conn_recv_block(struct qtfs_conn_var_s *pvar) +{ + int ret = 0; + ret = qtfs_conn_sync_magic(pvar, true); + if (ret != 0) { + return ret; + } + ret = do_qtfs_conn_recv(pvar, true); + if (ret > 0) { + pvar->recv_valid = (ret > pvar->recv_max) ? pvar->recv_max : ret; + } + return ret; +} + +int qtfs_conn_recv(struct qtfs_conn_var_s *pvar) +{ + int ret = 0; + ret = qtfs_conn_sync_magic(pvar, true); + if (ret != 0) { + return ret; + } + ret = do_qtfs_conn_recv(pvar, false); + if (ret <= 0) { + msleep(1); + } else { + pvar->recv_valid = (ret > pvar->recv_max) ? pvar->recv_max : ret; + } + return ret; +} + +int qtfs_conn_var_init(struct qtfs_conn_var_s *pvar) +{ + INIT_LIST_HEAD(&pvar->lst); + // qtfs消息为130多k，当作最大值作为合法性判断 + if (pvar->recv_max > QTFS_MSG_LEN || pvar->send_max > QTFS_MSG_LEN || + pvar->recv_max == 0 || pvar->recv_max == 0) { + qtfs_err("invalid recv max:%u or invalid send max:%u", + pvar->recv_max, pvar->send_max); + return QTFS_ERR; + } + pvar->vec_recv.iov_base = kmalloc(pvar->recv_max, GFP_KERNEL); + if (pvar->vec_recv.iov_base == NULL) { + qtfs_err("qtfs recv kmalloc failed, len:%u.\n", pvar->recv_max); + return QTFS_ERR; + } + pvar->vec_send.iov_base = kmalloc(pvar->send_max, GFP_KERNEL); + if (pvar->vec_send.iov_base == NULL) { + qtfs_err("qtfs send kmalloc failed, len:%u.\n", pvar->send_max); + kfree(pvar->vec_recv.iov_base); + pvar->vec_recv.iov_base = NULL; + return QTFS_ERR; + } + pvar->vec_recv.iov_len = pvar->recv_max; + pvar->vec_send.iov_len = 0; + memset(pvar->vec_recv.iov_base, 0, pvar->recv_max); + memset(pvar->vec_send.iov_base, 0, pvar->send_max); + pvar->recv_valid = 0; + pvar->send_valid = 0; + qtfs_info("init pvar thread:%d recv max:%u, send max:%u", pvar->cur_threadidx, pvar->recv_max, pvar->send_max); + return QTFS_OK; +} + +void qtfs_conn_var_fini(struct qtfs_conn_var_s *pvar) +{ + if (pvar->vec_recv.iov_base != NULL) { + kfree(pvar->vec_recv.iov_base); + pvar->vec_recv.iov_base = NULL; + pvar->vec_recv.iov_len = 0; + } + if (pvar->vec_send.iov_base != NULL) { + kfree(pvar->vec_send.iov_base); + pvar->vec_send.iov_base = NULL; + pvar->vec_send.iov_len = 0; + } + + return; +} + +void qtfs_conn_msg_clear(struct qtfs_conn_var_s *pvar) +{ + memset(pvar->vec_recv.iov_base, 0, pvar->recv_valid); + memset(pvar->vec_send.iov_base, 0, pvar->send_valid); + pvar->recv_valid = 0; + pvar->send_valid = 0; +#ifdef QTFS_CLIENT + memset(pvar->who_using, 0, QTFS_FUNCTION_LEN); +#endif + return; +} + +void *qtfs_conn_msg_buf(struct qtfs_conn_var_s *pvar, int dir) +{ + struct qtreq *req = (dir == QTFS_SEND) ? pvar->vec_send.iov_base : pvar->vec_recv.iov_base; + return req->data; +} + +// state machine +#define QTCONN_CUR_STATE(pvar) ((pvar->state == QTCONN_INIT) ? "INIT" : \ + ((pvar->state == QTCONN_CONNECTING) ? "CONNECTING" : \ + ((pvar->state == QTCONN_ACTIVE) ? "ACTIVE" : "UNKNOWN"))) + +static int qtfs_sm_connecting(struct qtfs_conn_var_s *pvar) +{ + int ret = QTERROR; + int retry = 3; + while (qtfs_mod_exiting == false && retry-- > 0) { + ret = pvar->conn_ops->conn_new_connection(&pvar->conn_var, pvar->user_type); + if (ret == 0) { + qtfs_info("qtfs sm connecting connect to a new connection."); + break; + } + msleep(100); + } + + return ret; +} + +int qtfs_sm_active(struct qtfs_conn_var_s *pvar) +{ + int ret = 0; + + switch (pvar->state) { + case QTCONN_ACTIVE: + // do nothing + break; + case QTCONN_INIT: + ret = qtfs_conn_init(pvar); + if (ret) { + qtfs_err("qtfs sm active init failed, ret:%d.", ret); + break; + } + // dont break, just enter connecting state to process + pvar->state = QTCONN_CONNECTING; + qtfs_info("qtfs sm active connecting, threadidx:%d", + pvar->cur_threadidx); + // fall-through + + case QTCONN_CONNECTING: + // accept(server) or connect(client) + ret = qtfs_sm_connecting(pvar); + if (ret == 0) + pvar->state = QTCONN_ACTIVE; + break; + default: + qtfs_err("qtfs sm active unknown state:%s.", QTCONN_CUR_STATE(pvar)); + ret = -EINVAL; + break; + } + return ret; +} + +int qtfs_sm_reconnect(struct qtfs_conn_var_s *pvar) +{ + int ret = QTOK; + switch (pvar->state) { + case QTCONN_INIT: + WARN_ON(1); + qtfs_err("qtfs sm reconnect state error!"); + ret = QTERROR; + break; + case QTCONN_ACTIVE: + qtfs_conn_fini(pvar); + ret = qtfs_conn_init(pvar); + if (ret) { + qtfs_err("qtfs sm active init failed, ret:%d.", ret); + ret = QTERROR; + pvar->state = QTCONN_INIT; + break; + } + + pvar->state = QTCONN_CONNECTING; + qtfs_warn("qtfs sm reconnect thread:%d, state:%s.", pvar->cur_threadidx, QTCONN_CUR_STATE(pvar)); + // fall-through + case QTCONN_CONNECTING: + ret = qtfs_sm_connecting(pvar); + if (ret == 0) + pvar->state = QTCONN_ACTIVE; + break; + default: + qtfs_err("qtfs sm reconnect unknown state:%s.", QTCONN_CUR_STATE(pvar)); + ret = QTERROR; + break; + } + return ret; +} + +int qtfs_sm_exit(struct qtfs_conn_var_s *pvar) +{ + int ret = QTOK; + switch (pvar->state) { + case QTCONN_INIT: + // do nothing + break; + case QTCONN_ACTIVE: + case QTCONN_CONNECTING: + qtfs_conn_fini(pvar); +#ifdef QTFS_SERVER + pvar->state = QTCONN_CONNECTING; +#endif +#ifdef QTFS_CLIENT + pvar->state = QTCONN_INIT; +#endif + qtfs_warn("qtfs sm exit thread:%d state:%s.", pvar->cur_threadidx, QTCONN_CUR_STATE(pvar)); + break; + + default: + qtfs_err("qtfs sm exit unknown state:%s.", QTCONN_CUR_STATE(pvar)); + ret = QTERROR; + break; + } + return ret; +} + +int qtfs_mutex_lock_interruptible(struct mutex *lock) +{ + int ret; + ret = mutex_lock_interruptible(lock); + if (ret == 0) { + // mutex lock successed, proc lazy put + while (1) { + struct llist_node *toput = llist_del_first(&g_lazy_put_llst); + struct qtfs_conn_var_s *pvar; + if (toput == NULL) + break; + pvar = llist_entry(toput, struct qtfs_conn_var_s, lazy_put); + pvar->conn_ops->conn_msg_clear(pvar); + list_move_tail(&pvar->lst, &g_vld_lst); + qtfs_warn("qtfs pvar lazy put idx:%d.", pvar->cur_threadidx); + } + } + return ret; +} + +static void parse_param(void) +{ + // reserve for pcie conn type + // default as socket type + g_pvar_ops = &qtfs_conn_sock_pvar_ops; + // calling conn specific parse_param + g_pvar_ops->parse_param(); +} + +int qtfs_conn_param_init(void) +{ +#ifdef QTFS_CLIENT + qtfs_fifo_pvar_cache = kmem_cache_create("qtfs_fifo_pvar", + sizeof(struct qtfs_conn_var_s), + 0, + (SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD), + NULL); + if (!qtfs_fifo_pvar_cache) { + qtfs_err("qtfs fifo pvar cache create failed.\n"); + return -ENOMEM; + } +#endif + INIT_LIST_HEAD(&g_vld_lst); + INIT_LIST_HEAD(&g_busy_lst); + INIT_LIST_HEAD(&g_fifo_lst); + init_llist_head(&g_lazy_put_llst); + atomic_set(&g_qtfs_conn_num, 0); + // parse module_param and choose specified channel + // should set g_pvar_ops here + parse_param(); + g_pvar_ops->param_init(); + + mutex_init(&g_param_mutex); + mutex_init(&g_fifo_mutex); + return 0; +} + +void release_pvar(struct qtfs_conn_var_s *pvar) +{ + if (!pvar) + return; + + pvar->conn_ops->conn_var_fini(pvar); + + qtfs_sm_exit(pvar); + if (pvar->cur_threadidx < 0 || pvar->cur_threadidx >= QTFS_MAX_THREADS) { + qtfs_err("qtfs free unknown threadidx %d", pvar->cur_threadidx); + } else { + qtfs_thread_var[pvar->cur_threadidx] = NULL; + qtfs_info("qtfs free pvar idx:%d successed.", pvar->cur_threadidx); + } + list_del(&pvar->lst); + kfree(pvar); +} + +void qtfs_conn_param_fini(void) +{ + struct list_head *plst; + struct list_head *n; + int ret; + int conn_num; + int i; + +#ifdef QTFS_CLIENT + kmem_cache_destroy(qtfs_fifo_pvar_cache); +#endif + + ret = qtfs_mutex_lock_interruptible(&g_param_mutex); + if (ret) { + qtfs_err("qtfs conn param finish mutex lock interrup failed, ret:%d.", ret); + WARN_ON(1); + return; + } + + list_for_each_safe(plst, n, &g_vld_lst) { + release_pvar((struct qtfs_conn_var_s *)plst); + } + list_for_each_safe(plst, n, &g_busy_lst) { + release_pvar((struct qtfs_conn_var_s *)plst); + } + + conn_num = atomic_read(&g_qtfs_conn_num); + for (i = 0; i < conn_num; i++) { + if (qtfs_thread_var[i] != NULL) { + qtfs_err("qtfs param not free idx:%d holder:%s", + qtfs_thread_var[i]->cur_threadidx, + qtfs_thread_var[i]->who_using); + } + } + mutex_unlock(&g_param_mutex); + g_pvar_ops->param_fini(); +} + +struct qtfs_conn_var_s *_qtfs_conn_get_param(const char *func) +{ + struct qtfs_conn_var_s *pvar = NULL; + int ret; + int cnt = 0; + + if (qtfs_mod_exiting == true) { + qtfs_warn("qtfs module is exiting, good bye!"); + return NULL; + } + +retry: + ret = qtfs_mutex_lock_interruptible(&g_param_mutex); + if (ret) { + qtfs_err("qtfs conn get param mutex lock interrup failed, ret:%d.", ret); + return NULL; + } + if (!list_empty(&g_vld_lst)) + pvar = list_last_entry(&g_vld_lst, struct qtfs_conn_var_s, lst); + if (pvar != NULL) { + list_move_tail(&pvar->lst, &g_busy_lst); + } + mutex_unlock(&g_param_mutex); + + if (pvar != NULL) { + int ret; + if (pvar->state == QTCONN_ACTIVE && pvar->conn_ops->conn_connected(&pvar->conn_var) == false) { + qtfs_warn("qtfs get param thread:%d disconnected, try to reconnect.", pvar->cur_threadidx); + ret = qtfs_sm_reconnect(pvar); + } else { + ret = qtfs_sm_active(pvar); + } + if (ret != 0) { + qtfs_conn_put_param(pvar); + return (IS_ERR_VALUE((long)ret) ? ERR_PTR((long)ret) : NULL); + } + strlcpy(pvar->who_using, func, QTFS_FUNCTION_LEN); + return pvar; + } + + ret = qtfs_mutex_lock_interruptible(&g_param_mutex); + if (ret) { + qtfs_err("qtfs conn get param mutex lock interrup failed, ret:%d.", ret); + return NULL; + } + if (atomic_read(&g_qtfs_conn_num) >= qtfs_conn_max_conn) { + mutex_unlock(&g_param_mutex); + cnt++; + msleep(1); + if (cnt < QTFS_GET_PARAM_MAX_RETRY) + goto retry; + qtfs_err("qtfs get param failed, the concurrency specification has reached the upper limit"); + return NULL; + } + pvar = kmalloc(sizeof(struct qtfs_conn_var_s), GFP_KERNEL); + if (pvar == NULL) { + qtfs_err("qtfs get param kmalloc failed.\n"); + mutex_unlock(&g_param_mutex); + return NULL; + } + memset(pvar, 0, sizeof(struct qtfs_conn_var_s)); + // initialize conn_pvar here + pvar->recv_max = QTFS_MSG_LEN; + pvar->send_max = QTFS_MSG_LEN; + pvar->user_type = QTFS_CONN_TYPE_QTFS; + g_pvar_ops->pvar_init(&pvar->conn_var, &pvar->conn_ops, pvar->user_type); + if (QTFS_OK != pvar->conn_ops->conn_var_init(pvar)) { + qtfs_err("qtfs sock var init failed.\n"); + kfree(pvar); + mutex_unlock(&g_param_mutex); + return NULL; + } + + memcpy(pvar->who_using, func, (strlen(func) >= QTFS_FUNCTION_LEN - 1) ? (QTFS_FUNCTION_LEN - 1) : strlen(func)); + pvar->cur_threadidx = atomic_read(&g_qtfs_conn_num); + qtfs_info("qtfs create new param, cur conn num:%d\n", atomic_read(&g_qtfs_conn_num)); + + qtfs_thread_var[pvar->cur_threadidx] = pvar; + // add to busy list + atomic_inc(&g_qtfs_conn_num); + list_add(&pvar->lst, &g_busy_lst); + + pvar->state = QTCONN_INIT; + pvar->seq_num = 0; + +#ifdef QTFS_CLIENT + mutex_unlock(&g_param_mutex); + ret = qtfs_sm_active(pvar); + if (ret) { + qtfs_err("qtfs get param active connection failed, ret:%d, curstate:%s", ret, QTCONN_CUR_STATE(pvar)); + // put to vld list + qtfs_conn_put_param(pvar); + return (IS_ERR_VALUE((long)ret) ? ERR_PTR((long)ret) : NULL); + } + qtfs_thread_var[pvar->cur_threadidx] = pvar; +#else + if (!pvar->conn_ops->conn_inited(pvar, pvar->user_type)) { + if ((ret = qtfs_sm_active(pvar)) != 0) { + qtfs_err("qtfs get param active connection failed, ret:%d, curstate:%s", ret, QTCONN_CUR_STATE(pvar)); + // put to vld list + mutex_unlock(&g_param_mutex); + qtfs_conn_put_param(pvar); + return (IS_ERR_VALUE((long)ret) ? ERR_PTR((long)ret) : NULL); + } + mutex_unlock(&g_param_mutex); + } else { + mutex_unlock(&g_param_mutex); + pvar->state = QTCONN_CONNECTING; + ret = qtfs_sm_active(pvar); + if (ret) { + qtfs_err("qtfs get param active connection failed, ret:%d curstate:%s", ret, QTCONN_CUR_STATE(pvar)); + qtfs_conn_put_param(pvar); + return (IS_ERR_VALUE((long)ret) ? ERR_PTR((long)ret) : NULL); + } + } +#endif + qtinfo_cntinc(QTINF_ACTIV_CONN); + + return pvar; +} + +struct qtfs_conn_var_s *qtfs_epoll_establish_conn(void) +{ + struct qtfs_conn_var_s *pvar = NULL; + int ret; + + pvar = qtfs_epoll_var; + if (pvar) { + if (pvar->state == QTCONN_ACTIVE && pvar->conn_ops->conn_connected(&pvar->conn_var) == false) { + qtfs_warn("qtfs epoll get param thread:%d disconnected, try to reconnect.", pvar->cur_threadidx); + ret = qtfs_sm_reconnect(pvar); + } else { + ret = qtfs_sm_active(pvar); + } + if (ret) { + return NULL; + } + return pvar; + } + + pvar = kmalloc(sizeof(struct qtfs_conn_var_s), GFP_KERNEL); + if (pvar == NULL) { + qtfs_err("qtfs get param kmalloc failed.\n"); + return NULL; + } + memset(pvar, 0, sizeof(struct qtfs_conn_var_s)); + pvar->recv_max = QTFS_EPOLL_MSG_LEN; + pvar->send_max = QTFS_EPOLL_MSG_LEN; + pvar->user_type = QTFS_CONN_TYPE_EPOLL; + pvar->cur_threadidx = QTFS_EPOLL_THREADIDX; + g_pvar_ops->pvar_init(&pvar->conn_var, &pvar->conn_ops, pvar->user_type); + if (QTFS_OK != pvar->conn_ops->conn_var_init(pvar)) { + qtfs_err("qtfs sock var init failed.\n"); + kfree(pvar); + return NULL; + } + qtfs_epoll_var = pvar; + pvar->state = QTCONN_INIT; + + ret = qtfs_sm_active(pvar); + if (ret) { + qtfs_err("qtfs epoll get param active new param failed, ret:%d state:%s", ret, QTCONN_CUR_STATE(pvar)); + return pvar; + } + + qtfs_info("qtfs create new epoll param state:%s", QTCONN_CUR_STATE(pvar)); + return pvar; +} + +void qtfs_conn_put_param(struct qtfs_conn_var_s *pvar) +{ + int ret; + if (!pvar) { + qtfs_err("qtfs_conn_var_s is null!!"); + return; + } + + ret = qtfs_mutex_lock_interruptible(&g_param_mutex); + if (ret) { + llist_add(&pvar->lazy_put, &g_lazy_put_llst); + qtfs_warn("qtfs conn put param add to lazy list idx:%d, ret:%d.", pvar->cur_threadidx, ret); + return; + } + pvar->conn_ops->conn_msg_clear(pvar); + list_move_tail(&pvar->lst, &g_vld_lst); + mutex_unlock(&g_param_mutex); +} + +void qtfs_epoll_cut_conn(struct qtfs_conn_var_s *pvar) +{ + int ret = 0; + + if (!pvar) { + qtfs_err("qtfs_conn_var_s is null!!"); + return; + } + + ret = qtfs_sm_exit(pvar); + if (ret) { + qtfs_err("qtfs epoll put param exit failed, ret:%d state:%s", ret, QTCONN_CUR_STATE(pvar)); + } +} + +#ifdef QTFS_CLIENT +/* fifo的机制有所不同，每一个pvar对应唯一一个fifo的访问，生命周期贯穿 + 从fifo open开始到fifo close结束，在open时get param，在close时put param */ +#define QTFS_FIFO_MAGIC_SEND 0xa55aa55a +#define QTFS_FIFO_MAGIC_RECV 0x5aa55aa5 +struct qtfs_conn_var_s *qtfs_fifo_get_param(void) +{ + int ret; + struct qtfs_conn_var_s *pvar = kmem_cache_alloc(qtfs_fifo_pvar_cache, GFP_KERNEL); + if (pvar == NULL) { + qtfs_err("kmem cache alloc fifo cache failed."); + return NULL; + } + memset(pvar, 0, sizeof(struct qtfs_conn_var_s)); + // initialize conn_pvar here + pvar->recv_max = QTFS_FIFO_REQ_LEN; + pvar->send_max = QTFS_FIFO_REQ_LEN; + pvar->magic_send = QTFS_FIFO_MAGIC_SEND; + pvar->magic_recv = QTFS_FIFO_MAGIC_RECV; + pvar->user_type = QTFS_CONN_TYPE_FIFO; + g_pvar_ops->pvar_init(&pvar->conn_var, &pvar->conn_ops, pvar->user_type); + if (QTFS_OK != pvar->conn_ops->conn_var_init(pvar)) { + qtfs_err("qtfs sock var init failed.\n"); + kmem_cache_free(qtfs_fifo_pvar_cache, pvar); + return NULL; + } + pvar->state = QTCONN_INIT; + + ret = qtfs_sm_active(pvar); + if (ret) { + qtfs_err("qtfs fifo get param active new param faile, ret:%d state:%s", ret, QTCONN_CUR_STATE(pvar)); + pvar->conn_ops->conn_var_fini(pvar); + kmem_cache_free(qtfs_fifo_pvar_cache, pvar); + return NULL; + } + mutex_lock(&g_fifo_mutex); + list_add(&pvar->lst, &g_fifo_lst); + mutex_unlock(&g_fifo_mutex); + qtfs_info("qtfs create new fifo param state:%s", QTCONN_CUR_STATE(pvar)); + return pvar; +} + +void qtfs_fifo_put_param(struct qtfs_conn_var_s *pvar) +{ + mutex_lock(&g_fifo_mutex); + list_del(&pvar->lst); + mutex_unlock(&g_fifo_mutex); + qtfs_sm_exit(pvar); + pvar->conn_ops->conn_var_fini(pvar); + kmem_cache_free(qtfs_fifo_pvar_cache, pvar); + return; +} +#endif + +void qtfs_conn_list_cnt(void) +{ + struct list_head *entry; + struct qtfs_conn_var_s *pvar; +#ifdef QTFS_CLIENT + int ret = 0; + ret = qtfs_mutex_lock_interruptible(&g_param_mutex); + if (ret) { + qtfs_err("qtfs conn put param mutex lock interrup failed, ret:%d.", ret); + return; + } +#endif + qtfs_diag_info->pvar_busy = 0; + qtfs_diag_info->pvar_vld = 0; + memset(qtfs_diag_info->who_using, 0, sizeof(qtfs_diag_info->who_using)); + list_for_each(entry, &g_busy_lst) { + qtfs_diag_info->pvar_busy++; + pvar = (struct qtfs_conn_var_s *)entry; + if (pvar->cur_threadidx < 0 || pvar->cur_threadidx >= QTFS_MAX_THREADS) + continue; + strlcpy(qtfs_diag_info->who_using[pvar->cur_threadidx], + qtfs_thread_var[pvar->cur_threadidx]->who_using, QTFS_FUNCTION_LEN); + } + list_for_each(entry, &g_vld_lst) + qtfs_diag_info->pvar_vld++; +#ifdef QTFS_CLIENT + mutex_unlock(&g_param_mutex); +#endif +} + +module_param(qtfs_conn_max_conn, int, 0600); +module_param_string(qtfs_log_level, qtfs_log_level, sizeof(qtfs_log_level), 0600); +module_param_string(qtfs_conn_type, qtfs_conn_type, sizeof(qtfs_conn_type), 0600); \ No newline at end of file diff --git a/qtfs/qtfs_common/misc.c b/qtfs/qtfs_common/misc.c new file mode 100644 index 0000000..6f84b95 --- /dev/null +++ b/qtfs/qtfs_common/misc.c @@ -0,0 +1,355 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "comm.h" +#include "log.h" +#include "req.h" +#include "conn.h" + +struct qtfs_wl g_qtfs_wl; + +extern struct file_operations qtfs_misc_fops; +struct mutex qtfs_diag_info_lock; + +static struct miscdevice qtfs_misc_dev = { + .minor = MISC_DYNAMIC_MINOR, +#ifndef QTFS_CLIENT + .name = "qtfs_server", +#else + .name = "qtfs_client", +#endif + .fops = &qtfs_misc_fops, +}; + +int qtfs_misc_register(void) +{ + int ret = misc_register(&qtfs_misc_dev); + if (ret) { + qtfs_err("qtfs misc register failed, ret:%d.", ret); + return -EFAULT; + } + mutex_init(&qtfs_diag_info_lock); + return 0; +} + +void qtfs_misc_destroy(void) +{ + misc_deregister(&qtfs_misc_dev); + return; +} + +void qtfs_misc_flush_threadstate(void) +{ + int i; + for (i = 0; i < QTFS_MAX_THREADS; i++) { + if (qtfs_thread_var[i] == NULL) { + qtfs_diag_info->thread_state[i] = -1; + continue; + } + qtfs_diag_info->thread_state[i] = qtfs_thread_var[i]->state; + } + qtfs_diag_info->epoll_state = (qtfs_epoll_var == NULL) ? -1 : qtfs_epoll_var->state; +} + +void qtfs_req_size(void) +{ + qtfs_diag_info->req_size[QTFS_REQ_NULL] = sizeof(struct qtreq); + qtfs_diag_info->req_size[QTFS_REQ_IOCTL] = sizeof(struct qtreq_ioctl); + qtfs_diag_info->req_size[QTFS_REQ_STATFS] = sizeof(struct qtreq_statfs); + qtfs_diag_info->req_size[QTFS_REQ_MOUNT] = sizeof(struct qtreq_mount); + qtfs_diag_info->req_size[QTFS_REQ_OPEN] = sizeof(struct qtreq_open); + qtfs_diag_info->req_size[QTFS_REQ_CLOSE] = sizeof(struct qtreq_close); + qtfs_diag_info->req_size[QTFS_REQ_READITER] = sizeof(struct qtreq_readiter); + qtfs_diag_info->req_size[QTFS_REQ_WRITE] = sizeof(struct qtreq_write); + qtfs_diag_info->req_size[QTFS_REQ_LOOKUP] = sizeof(struct qtreq_lookup); + qtfs_diag_info->req_size[QTFS_REQ_READDIR] = sizeof(struct qtreq_readdir); + qtfs_diag_info->req_size[QTFS_REQ_MKDIR] = sizeof(struct qtreq_mkdir); + qtfs_diag_info->req_size[QTFS_REQ_RMDIR] = sizeof(struct qtreq_rmdir); + qtfs_diag_info->req_size[QTFS_REQ_GETATTR] = sizeof(struct qtreq_getattr); + qtfs_diag_info->req_size[QTFS_REQ_SETATTR] = sizeof(struct qtreq_setattr); + qtfs_diag_info->req_size[QTFS_REQ_ICREATE] = sizeof(struct qtreq_icreate); + qtfs_diag_info->req_size[QTFS_REQ_MKNOD] = sizeof(struct qtreq_mknod); + qtfs_diag_info->req_size[QTFS_REQ_UNLINK] = sizeof(struct qtreq_unlink); + qtfs_diag_info->req_size[QTFS_REQ_SYMLINK] = sizeof(struct qtreq_symlink); + qtfs_diag_info->req_size[QTFS_REQ_LINK] = sizeof(struct qtreq_link); + qtfs_diag_info->req_size[QTFS_REQ_GETLINK] = sizeof(struct qtreq_getlink); + qtfs_diag_info->req_size[QTFS_REQ_RENAME] = sizeof(struct qtreq_rename); + qtfs_diag_info->req_size[QTFS_REQ_XATTRLIST] = sizeof(struct qtreq_xattrlist); + qtfs_diag_info->req_size[QTFS_REQ_XATTRGET] = sizeof(struct qtreq_xattrget); + qtfs_diag_info->req_size[QTFS_REQ_SYSMOUNT] = sizeof(struct qtreq_sysmount); + qtfs_diag_info->req_size[QTFS_REQ_SYSUMOUNT] = sizeof(struct qtreq_sysumount); + qtfs_diag_info->req_size[QTFS_REQ_FIFOPOLL] = sizeof(struct qtreq_poll); + qtfs_diag_info->req_size[QTFS_REQ_EPOLL_CTL] = sizeof(struct qtreq_epollctl); + qtfs_diag_info->req_size[QTFS_REQ_EPOLL_EVENT] = sizeof(struct qtreq_epollevt); + + qtfs_diag_info->rsp_size[QTFS_REQ_NULL] = sizeof(struct qtreq); + qtfs_diag_info->rsp_size[QTFS_REQ_IOCTL] = sizeof(struct qtrsp_ioctl); + qtfs_diag_info->rsp_size[QTFS_REQ_STATFS] = sizeof(struct qtrsp_statfs); + qtfs_diag_info->rsp_size[QTFS_REQ_MOUNT] = sizeof(struct qtrsp_mount); + qtfs_diag_info->rsp_size[QTFS_REQ_OPEN] = sizeof(struct qtrsp_open); + qtfs_diag_info->rsp_size[QTFS_REQ_CLOSE] = sizeof(struct qtrsp_close); + qtfs_diag_info->rsp_size[QTFS_REQ_READITER] = sizeof(struct qtrsp_readiter); + qtfs_diag_info->rsp_size[QTFS_REQ_WRITE] = sizeof(struct qtrsp_write); + qtfs_diag_info->rsp_size[QTFS_REQ_LOOKUP] = sizeof(struct qtrsp_lookup); + qtfs_diag_info->rsp_size[QTFS_REQ_READDIR] = sizeof(struct qtrsp_readdir); + qtfs_diag_info->rsp_size[QTFS_REQ_MKDIR] = sizeof(struct qtrsp_mkdir); + qtfs_diag_info->rsp_size[QTFS_REQ_RMDIR] = sizeof(struct qtrsp_rmdir); + qtfs_diag_info->rsp_size[QTFS_REQ_GETATTR] = sizeof(struct qtrsp_getattr); + qtfs_diag_info->rsp_size[QTFS_REQ_SETATTR] = sizeof(struct qtrsp_setattr); + qtfs_diag_info->rsp_size[QTFS_REQ_ICREATE] = sizeof(struct qtrsp_icreate); + qtfs_diag_info->rsp_size[QTFS_REQ_MKNOD] = sizeof(struct qtrsp_mknod); + qtfs_diag_info->rsp_size[QTFS_REQ_UNLINK] = sizeof(struct qtrsp_unlink); + qtfs_diag_info->rsp_size[QTFS_REQ_SYMLINK] = sizeof(struct qtrsp_symlink); + qtfs_diag_info->rsp_size[QTFS_REQ_LINK] = sizeof(struct qtrsp_link); + qtfs_diag_info->rsp_size[QTFS_REQ_GETLINK] = sizeof(struct qtrsp_getlink); + qtfs_diag_info->rsp_size[QTFS_REQ_RENAME] = sizeof(struct qtrsp_rename); + qtfs_diag_info->rsp_size[QTFS_REQ_XATTRLIST] = sizeof(struct qtrsp_xattrlist); + qtfs_diag_info->rsp_size[QTFS_REQ_XATTRGET] = sizeof(struct qtrsp_xattrget); + qtfs_diag_info->rsp_size[QTFS_REQ_SYSMOUNT] = sizeof(struct qtrsp_sysmount); + qtfs_diag_info->rsp_size[QTFS_REQ_SYSUMOUNT] = sizeof(struct qtrsp_sysumount); + qtfs_diag_info->rsp_size[QTFS_REQ_FIFOPOLL] = sizeof(struct qtrsp_poll); + qtfs_diag_info->rsp_size[QTFS_REQ_EPOLL_CTL] = sizeof(struct qtrsp_epollctl); + qtfs_diag_info->rsp_size[QTFS_REQ_EPOLL_EVENT] = sizeof(struct qtrsp_epollevt); +} + +void qtfs_whitelist_initset(void) +{ + int type; + rwlock_init(&g_qtfs_wl.rwlock); + for (type = 0; type < QTFS_WHITELIST_MAX; type++) { + memset(&g_qtfs_wl.cap[type], 0, sizeof(struct qtfs_wl_cap)); + } + return; +} + +static int qtfs_whitelist_dup_check(char *tar, struct qtfs_wl_cap *cap) +{ + int i; + for (i = 0; i < cap->nums; i++) { + if (strncmp(tar, cap->item[i], QTFS_PATH_MAX - 1) == 0) + return 1; + } + return 0; +} + +static int qtfs_whitelist_add(struct qtfs_wl_item *uitem) +{ + // uitem->type is checked + struct qtfs_wl_cap *cap = &g_qtfs_wl.cap[uitem->type]; + write_lock(&g_qtfs_wl.rwlock); + if (cap->nums >= QTFS_WL_MAX_NUM) { + qtfs_err("qtfs add white list failed, nums:%u reach upper limit:%d", cap->nums, QTFS_WL_MAX_NUM); + goto err_end; + } + cap->item[cap->nums] = (char *)kmalloc(uitem->len + 1, GFP_KERNEL); + if (IS_ERR_OR_NULL(cap->item[cap->nums])) { + qtfs_err("kmalloc error"); + goto err_end; + } + memset(cap->item[cap->nums], 0, uitem->len + 1); + if (copy_from_user(cap->item[cap->nums], uitem->path, uitem->len) || + qtfs_whitelist_dup_check(cap->item[cap->nums], cap) == 1) { + qtfs_err("copy from user failed or item is a duplicate, len:%u type:%u", uitem->len, uitem->type); + kfree(cap->item[cap->nums]); + cap->item[cap->nums] = NULL; + goto err_end; + } + qtfs_info("Successed to add white list type:%u len:%u path:[%s]", uitem->type, uitem->len, cap->item[cap->nums]); + cap->nums++; + write_unlock(&g_qtfs_wl.rwlock); + return 0; + +err_end: + write_unlock(&g_qtfs_wl.rwlock); + return -1; +} + +static int qtfs_whitelist_del(struct qtfs_wl_item *uitem) +{ + // type is checked + struct qtfs_wl_cap *cap = &g_qtfs_wl.cap[uitem->type]; + write_lock(&g_qtfs_wl.rwlock); + if (uitem->index >= cap->nums) { + qtfs_err("White list del type:%u nums:%u, invalid index:%u", uitem->type, cap->nums, uitem->index); + goto err_end; + } + // free target index + kfree(cap->item[uitem->index]); + cap->item[uitem->index] = NULL; + if (cap->nums > 1) { + // if nums > 1 move last one to fill the hole + cap->item[uitem->index] = cap->item[cap->nums - 1]; + cap->item[cap->nums - 1] = NULL; + } + qtfs_info("white list del type:%u total nums:%u delindex:%u", uitem->type, cap->nums, uitem->index); + + cap->nums--; + write_unlock(&g_qtfs_wl.rwlock); + return 0; +err_end: + write_unlock(&g_qtfs_wl.rwlock); + return -1; +} + +static int qtfs_whitelist_get(struct qtfs_wl_item *uitem) +{ + // type is checked + struct qtfs_wl_cap *cap = &g_qtfs_wl.cap[uitem->type]; + int len; + read_lock(&g_qtfs_wl.rwlock); + if (uitem->index >= cap->nums) { + qtfs_err("query white list invalid index:%u type:%u total nums:%u", uitem->index, uitem->type, cap->nums); + goto err_end; + } + len = strlen(cap->item[uitem->index]); + if (!access_ok(uitem->path, len)) { + qtfs_err("white list query pointer of userspace is not valid type:%u index:%u len:%d", uitem->type, uitem->index, len); + goto err_end; + } + if (copy_to_user(uitem->path, cap->item[uitem->index], len)) { + qtfs_err("white list query copy to user failed, type:%u index:%u len:%d", uitem->type, uitem->index, len); + goto err_end; + } + qtfs_info("white list query type:%u total nums:%u index:%u len:%d", uitem->type, cap->nums, uitem->index, len); + read_unlock(&g_qtfs_wl.rwlock); + return 0; + +err_end: + read_unlock(&g_qtfs_wl.rwlock); + return -1; +} + +void qtfs_whitelist_clearall(void) +{ + struct qtfs_wl_cap *cap = NULL; + int type; + int item; + write_lock(&g_qtfs_wl.rwlock); + for (type = 0; type < QTFS_WHITELIST_MAX; type++) { + cap = &g_qtfs_wl.cap[type]; + for (item = 0; item < cap->nums; item++) { + kfree(cap->item[item]); + cap->item[item] = NULL; + } + cap->nums = 0; + } + write_unlock(&g_qtfs_wl.rwlock); + return; +} + +void qtfs_whitelist_exit(void) +{ + qtfs_whitelist_clearall(); + return; +} + +long qtfs_misc_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + long ret = QTOK; + qtfs_info("qtfs client misc ioctl."); + switch (cmd) { + case QTFS_IOCTL_ALLINFO: + mutex_lock(&qtfs_diag_info_lock); + if (qtfs_diag_info == NULL) { + qtfs_err("ioctl allinfo failed, qtfs_diag_info is invalid."); + mutex_unlock(&qtfs_diag_info_lock); + goto err_end; + } + qtfs_req_size(); + qtfs_diag_info->log_level = log_level; + qtfs_misc_flush_threadstate(); + qtfs_conn_list_cnt(); + if (copy_to_user((void *)arg, qtfs_diag_info, sizeof(struct qtinfo))) { + qtfs_err("ioctl allinfo copy to user failed."); + mutex_unlock(&qtfs_diag_info_lock); + goto err_end; + } + mutex_unlock(&qtfs_diag_info_lock); + break; + case QTFS_IOCTL_CLEARALL: + mutex_lock(&qtfs_diag_info_lock); + qtinfo_clear(); + mutex_unlock(&qtfs_diag_info_lock); + break; + case QTFS_IOCTL_LOGLEVEL: { + char level_str[QTFS_LOGLEVEL_STRLEN] = {0}; + if (arg == 0 || copy_from_user(level_str, (void *)arg, QTFS_LOGLEVEL_STRLEN - 1)) { + qtfs_err("ioctl set log level failed, arg:%lu.", arg); + goto err_end; + } + ret = (long)qtfs_log_init(level_str, QTFS_LOGLEVEL_STRLEN); + break; + } + case QTFS_IOCTL_EPOLL_SUPPORT: + if (arg == 0) { + qtfs_epoll_mode = false; + } else { + qtfs_epoll_mode = true; + } + break; + case QTFS_IOCTL_WL_ADD: { + struct qtfs_wl_item head; + if (copy_from_user(&head, (void *)arg, sizeof(struct qtfs_wl_item))) { + qtfs_err("ioctl wl add copy from user failed"); + goto err_end; + } + if (head.len == 0 || head.len == MAX_PATH_LEN || head.type >= QTFS_WHITELIST_MAX || + !access_ok(head.path, head.len)) { + qtfs_err("ioctl wl add len:%u type:%u invalid", head.len, head.type); + goto err_end; + } + if (qtfs_whitelist_add(&head) != 0) { + qtfs_err("ioctl wl add failed!"); + goto err_end; + } + break; + } + case QTFS_IOCTL_WL_DEL: + case QTFS_IOCTL_WL_GET: { + struct qtfs_wl_item head; + if (copy_from_user(&head, (void *)arg, sizeof(struct qtfs_wl_item))) { + qtfs_err("ioctl wl del copy from user failed"); + goto err_end; + } + if (head.type >= QTFS_WHITELIST_MAX) { + qtfs_err("ioctl wl del invalid type:%u", head.type); + goto err_end; + } + if (cmd == QTFS_IOCTL_WL_DEL) { + if (qtfs_whitelist_del(&head) != 0) { + qtfs_err("ioctl wl del failed!"); + goto err_end; + } + } else { + if (qtfs_whitelist_get(&head) != 0) { + qtfs_err("ioctl wl get failed!"); + goto err_end; + } + } + break; + } + } + return ret; +err_end: + return QTERROR; +} diff --git a/qtfs/qtfs_common/qtfs_check.c b/qtfs/qtfs_common/qtfs_check.c new file mode 100644 index 0000000..13d993c --- /dev/null +++ b/qtfs/qtfs_common/qtfs_check.c @@ -0,0 +1,332 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + + +#include "req.h" +#include "qtfs_check.h" + +/* + 检查原则： + 1. 基本数据类型，据实严格判断合法范围，有数组、指针操作的注意数组越界或指针飞踩； + 2. 字符串类型，基本的判断在长度范围内要有结束符，多个字符串拼接的，每一段起始和 + 结束都能被结束符分开。 + +*/ + +// string类型基本防护，在max范围内最后一个字符必须是结束符，防止越界访问 +static inline bool check_string(char *str, size_t max) +{ + if (max == 0) + return false; + if (str[max - 1] != '\0') + return true; + return false; +} + +static inline bool check_fd(int fd) +{ +#define FILENO_STDERR 2 + if (fd <= FILENO_STDERR) + return true; + return false; +} + +#define TOREQ (typeof(req))in; +#define TORSP (typeof(rsp))in; +int req_check_none(void *in) +{ + return QTFS_CHECK_OK; +} + +int req_check_mount(void *in) +{ + struct qtreq_mount *req = TOREQ; + + if (check_string(req->path, sizeof(req->path))) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_open(void *in) +{ + struct qtreq_open *req = TOREQ; + + // flags 和 mode如果错误syscall会报错，不会有安全风险 + if (check_string(req->path, sizeof(req->path))) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_close(void *in) +{ + struct qtreq_close *req = TOREQ; + if (check_fd(req->fd)) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_readiter(void *in) +{ + struct qtreq_readiter *req = TOREQ; + if (check_fd(req->fd)) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_write(void *in) +{ + struct qtreq_write *req = TOREQ; + if (check_fd(req->d.fd) || req->d.buflen > sizeof(req->path_buf)) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_lookup(void *in) +{ + struct qtreq_lookup *req = TOREQ; + if (check_string(req->fullname, sizeof(req->fullname))) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_readdir(void *in) +{ + struct qtreq_readdir *req = TOREQ; + struct qtrsp_readdir *rsp = TORSP; + if (req->count != sizeof(rsp->dirent) || check_string(req->path, sizeof(req->path))) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_mkdir(void *in) +{ + struct qtreq_mkdir *req = TOREQ; + if (check_string(req->path, sizeof(req->path))) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_rmdir(void *in) +{ + struct qtreq_rmdir *req = TOREQ; + if (check_string(req->path, sizeof(req->path))) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_getattr(void *in) +{ + struct qtreq_getattr *req = TOREQ; + if (check_string(req->path, sizeof(req->path))) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_setattr(void *in) +{ + struct qtreq_setattr *req = TOREQ; + if (check_string(req->path, sizeof(req->path)) || req->attr.ia_file != NULL) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_icreate(void *in) +{ + struct qtreq_icreate *req = TOREQ; + if (check_string(req->path, sizeof(req->path))) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_mknod(void *in) +{ + struct qtreq_mknod *req = TOREQ; + if (check_string(req->path, sizeof(req->path))) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_unlink(void *in) +{ + struct qtreq_unlink *req = TOREQ; + if (check_string(req->path, sizeof(req->path))) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_symlink(void *in) +{ + struct qtreq_symlink *req = TOREQ; + int total_len = sizeof(req->path); + if (req->d.newlen + req->d.oldlen >= total_len || + req->d.newlen == 0 || req->d.oldlen == 0) + return QTFS_CHECK_ERR; + if (check_string(req->path, req->d.newlen) || + check_string(&req->path[req->d.newlen], req->d.oldlen)) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_link(void *in) +{ + struct qtreq_link *req = TOREQ; + int total_len = sizeof(req->path); + if (req->d.oldlen + req->d.newlen > total_len || + req->d.oldlen == 0 || req->d.newlen == 0) + return QTFS_CHECK_ERR; + if (check_string(req->path, req->d.oldlen) || + check_string(&req->path[req->d.oldlen], req->d.oldlen)) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_getlink(void *in) +{ + struct qtreq_getlink *req = TOREQ; + if (check_string(req->path, sizeof(req->path))) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_rename(void *in) +{ + struct qtreq_rename *req = TOREQ; + int total_len = sizeof(req->path); + if (req->d.oldlen + req->d.newlen > total_len || + req->d.oldlen == 0 || req->d.newlen == 0) + return QTFS_CHECK_ERR; + if (check_string(req->path, req->d.oldlen) || + check_string(&req->path[req->d.oldlen], req->d.oldlen)) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} +int req_check_xattrlist(void *in) +{ + struct qtreq_xattrlist *req = TOREQ; + if (check_string(req->path, sizeof(req->path))) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_xattrget(void *in) +{ + struct qtreq_xattrget *req = TOREQ; + if (check_string(req->d.prefix_name, sizeof(req->d.prefix_name)) || + check_string(req->path, PATH_MAX)) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_xattrset(void *in) +{ + struct qtreq_xattrset *req = TOREQ; + if (req->d.pathlen == 0 || req->d.namelen == 0 || + req->d.pathlen + req->d.namelen + req->d.valuelen > sizeof(req->buf)) + return QTFS_CHECK_ERR; + if (check_string(req->buf, req->d.pathlen) || + check_string(&req->buf[req->d.pathlen], req->d.namelen) || + check_string(&req->buf[req->d.pathlen + req->d.namelen], req->d.valuelen)) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_sysmount(void *in) +{ + int pos = 0; + struct qtreq_sysmount *req = TOREQ; + if (req->d.dev_len + req->d.dir_len + req->d.type_len + req->d.data_len > sizeof(req->buf)) + return QTFS_CHECK_ERR; + if (req->d.dir_len == 0) + return QTFS_CHECK_ERR; + if (check_string(req->buf, req->d.dev_len)) + return QTFS_CHECK_ERR; + pos += req->d.dev_len; + if (check_string(&req->buf[pos], req->d.dir_len)) + return QTFS_CHECK_ERR; + pos += req->d.dir_len; + if (check_string(&req->buf[pos], req->d.type_len)) + return QTFS_CHECK_ERR; + pos += req->d.type_len; + if (check_string(&req->buf[pos], req->d.data_len)) + return QTFS_CHECK_ERR; + + return QTFS_CHECK_OK; +} + +int req_check_sysumount(void *in) +{ + struct qtreq_sysumount *req = TOREQ; + if (check_string(req->buf, sizeof(req->buf))) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_fifopoll(void *in) +{ + struct qtreq_poll *req = TOREQ; + if (check_fd(req->fd)) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_statfs(void *in) +{ + struct qtreq_statfs *req = TOREQ; + if (check_string(req->path, sizeof(req->path))) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_ioctl(void *in) +{ + struct qtreq_ioctl *req = TOREQ; + if (req->d.argtype != 0 && req->d.argtype != 1) + return QTFS_CHECK_ERR; + if (req->d.size > sizeof(req->path) || check_fd(req->d.fd)) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_epoll_ctl(void *in) +{ + struct qtreq_epollctl *req = TOREQ; + if (check_fd(req->fd)) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_llseek(void *in) +{ + struct qtreq_llseek *req = TOREQ; + if (check_fd(req->fd)) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_sc_kill(void *in) +{ + return QTFS_CHECK_OK; +} + +int req_check_sc_sched_getaffinity(void *in) +{ + struct qtreq_sc_sched_affinity *req = TOREQ; + if (req->len > AFFINITY_MAX_LEN || req->len == 0) + return QTFS_CHECK_ERR; + return QTFS_CHECK_OK; +} + +int req_check_sc_sched_setaffinity(void *in) +{ + return req_check_sc_sched_getaffinity(in); +} diff --git a/qtfs/qtfs_common/socket.c b/qtfs/qtfs_common/socket.c new file mode 100644 index 0000000..c7e8d8f --- /dev/null +++ b/qtfs/qtfs_common/socket.c @@ -0,0 +1,544 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include +#include +#include +#include +#include + +#include "comm.h" +#include "conn.h" +#include "log.h" +#include "req.h" +#include "symbol_wrapper.h" + +#ifdef QTFS_TEST_MODE +char qtfs_server_ip[20] = "127.0.0.1"; +int qtfs_server_port = 12345; +#else +unsigned int qtfs_server_vsock_port = 12345; +unsigned int qtfs_server_vsock_cid = 2; // host cid in vm is always 2 +#endif + +#ifdef QTFS_SERVER +struct qtfs_main_sock_s { + struct socket *sock; + struct mutex lock; +}; +struct qtfs_main_sock_s qtfs_server_main_sock[QTFS_CONN_TYPE_INV]; + +#endif + +#ifdef KVER_4_19 +static inline void sock_valbool_flag(struct sock *sk, enum sock_flags bit, + int valbool) +{ + if (valbool) + sock_set_flag(sk, bit); + else + sock_reset_flag(sk, bit); +} + +void sock_set_keepalive(struct sock *sk) +{ + lock_sock(sk); + if (sk->sk_prot->keepalive) + sk->sk_prot->keepalive(sk, true); + sock_valbool_flag(sk, SOCK_KEEPOPEN, true); + release_sock(sk); +} + +int tcp_sock_set_keepidle_locked(struct sock *sk, int val) +{ + struct tcp_sock *tp = tcp_sk(sk); + + if (val < 1 || val > MAX_TCP_KEEPIDLE) + return -EINVAL; + + tp->keepalive_time = val * HZ; + if (sock_flag(sk, SOCK_KEEPOPEN) && + !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) { + u32 elapsed = keepalive_time_elapsed(tp); + + if (tp->keepalive_time > elapsed) + elapsed = tp->keepalive_time - elapsed; + else + elapsed = 0; + inet_csk_reset_keepalive_timer(sk, elapsed); + } + + return 0; +} + +int tcp_sock_set_keepidle(struct sock *sk, int val) +{ + int err; + + lock_sock(sk); + err = tcp_sock_set_keepidle_locked(sk, val); + release_sock(sk); + return err; +} + +int tcp_sock_set_keepintvl(struct sock *sk, int val) +{ + if (val < 1 || val > MAX_TCP_KEEPINTVL) + return -EINVAL; + + lock_sock(sk); + tcp_sk(sk)->keepalive_intvl = val * HZ; + release_sock(sk); + return 0; +} + +int tcp_sock_set_keepcnt(struct sock *sk, int val) +{ + if (val < 1 || val > MAX_TCP_KEEPCNT) + return -EINVAL; + + lock_sock(sk); + tcp_sk(sk)->keepalive_probes = val; + release_sock(sk); + return 0; +} + +void sock_set_reuseaddr(struct sock *sk) +{ + lock_sock(sk); + sk->sk_reuse = SK_CAN_REUSE; + release_sock(sk); +} +#endif + +#define QTSOCK_SET_KEEPX(sock, val) sock_set_keepalive(sock->sk); tcp_sock_set_keepcnt(sock->sk, val);\ + tcp_sock_set_keepidle(sock->sk, val); tcp_sock_set_keepintvl(sock->sk, val); + +static int qtfs_conn_sock_recv(void *connvar, void *buf, size_t len, bool block); +static int qtfs_conn_sock_send(void *connvar, void *buf, size_t len); +static void qtfs_conn_sock_fini(void *connvar, qtfs_conn_type_e type); + +void qtfs_sock_recvtimeo_set(struct socket *sock, __s64 sec, __s64 usec) +{ + int error; +#ifdef KVER_4_19 + struct timeval tv; +#else + struct __kernel_sock_timeval tv; + sockptr_t optval = KERNEL_SOCKPTR((void *)&tv); +#endif + tv.tv_sec = sec; + tv.tv_usec = usec; + + if (sock == NULL) { + qtfs_err("qtfs sock recvtimeo set failed, sock is invalid."); + return; + } + +#ifdef KVER_4_19 + error = sock_setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, + (char *)&tv, sizeof(tv)); +#else + error = sock_setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO_OLD, + optval, sizeof(struct __kernel_sock_timeval)); +#endif + if (error) { + qtfs_err("qtfs param setsockopt error, ret:%d.\n", error); + } +} + +#ifdef QTFS_SERVER +static int qtfs_conn_sock_server_accept(void *connvar, qtfs_conn_type_e type) +{ + struct qtfs_sock_var_s *sockvar = (struct qtfs_sock_var_s *)connvar; + int ret; + struct socket *sock = NULL; + if (type >= QTFS_CONN_TYPE_INV) { + qtfs_err("invalid type:%u.", type); + return -EFAULT; + } + mutex_lock(&qtfs_server_main_sock[type].lock); + if (qtfs_server_main_sock[type].sock == NULL) { + qtfs_err("invalid main sock, type:%u.", type); + mutex_unlock(&qtfs_server_main_sock[type].lock); + return -EFAULT; + } + sock = qtfs_server_main_sock[type].sock; + + if (sock == NULL) { + WARN_ON(1); + qtfs_err("qtfs server accept failed, main sock is NULL."); + mutex_unlock(&qtfs_server_main_sock[type].lock); + return -EINVAL; + } + ret = kernel_accept(sock, &sockvar->client_sock, SOCK_NONBLOCK); + if (ret < 0) { + mutex_unlock(&qtfs_server_main_sock[type].lock); + return ret; + } +#ifdef QTFS_TEST_MODE + QTSOCK_SET_KEEPX(sock, 5); +#else + sock_set_keepalive(sock->sk); +#endif + + qtfs_info("qtfs accept a client connection.\n"); + qtfs_sock_recvtimeo_set(sockvar->client_sock, QTFS_SOCK_RCVTIMEO, 0); + mutex_unlock(&qtfs_server_main_sock[type].lock); + return 0; +} + +static int qtfs_conn_sock_init(void *connvar, qtfs_conn_type_e type) +{ + struct qtfs_sock_var_s *sockvar = (struct qtfs_sock_var_s *)connvar; + struct socket *sock; + int ret = 0; + int sock_family = AF_VSOCK; +#ifdef QTFS_TEST_MODE + struct sockaddr_in saddr; + sock_family = AF_INET; + saddr.sin_family = sock_family; + saddr.sin_port = htons(sockvar->port); + saddr.sin_addr.s_addr = in_aton(sockvar->addr); +#else + struct sockaddr_vm saddr; + sock_family = AF_VSOCK; + saddr.svm_family = sock_family; + saddr.svm_port = sockvar->vm_port; + saddr.svm_cid = sockvar->vm_cid; +#endif + if (type >= QTFS_CONN_TYPE_INV) { + qtfs_err("invalid type in sock init:%u", type); + ret = -EINVAL; + goto err_end_type; + } + mutex_lock(&qtfs_server_main_sock[type].lock); + if (qtfs_server_main_sock[type].sock != NULL) { + qtfs_info("qtfs conn type:%u main sock is set, valid or out-of-date?", type); + mutex_unlock(&qtfs_server_main_sock[type].lock); + return 0; + } + + qtfs_info("qtfs sock server init enter"); + + ret = sock_create_kern(&init_net, sock_family, SOCK_STREAM, 0, &sock); + if (ret) { + qtfs_err("qtfs sock server init create sock failed.\n"); + goto err_end; + } + + sock_set_reuseaddr(sock->sk); +#ifdef QTFS_TEST_MODE + QTSOCK_SET_KEEPX(sock, 5); +#else + sock_set_keepalive(sock->sk); +#endif + + ret = sock->ops->bind(sock, (struct sockaddr *)&saddr, sizeof(saddr)); + if (ret < 0) { + qtfs_err("qtfs sock server bind error: %d.\n", ret); + goto err_end; + } + + ret = sock->ops->listen(sock, QTFS_SERVER_MAXCONN); + if (ret < 0) { + qtfs_err("qtfs sock server listen failed.\n"); + goto err_end; + } + + qtfs_info("qtfs socket init sock OK!"); + qtfs_server_main_sock[type].sock = sock; + mutex_unlock(&qtfs_server_main_sock[type].lock); + return 0; + +err_end: + mutex_unlock(&qtfs_server_main_sock[type].lock); + sock_release(sock); +err_end_type: + return ret; +} +#endif +#ifdef QTFS_CLIENT +static int qtfs_conn_sock_client_connect(void *connvar, qtfs_conn_type_e type) +{ + struct qtfs_sock_var_s *sockvar = (struct qtfs_sock_var_s *)connvar; + struct socket *sock = sockvar->client_sock; + int ret; +#ifdef QTFS_TEST_MODE + struct sockaddr_in saddr; + saddr.sin_family = AF_INET; + saddr.sin_port = htons(sockvar->port); + saddr.sin_addr.s_addr = in_aton(sockvar->addr); +#else + struct sockaddr_vm saddr; + saddr.svm_family = AF_VSOCK; + saddr.svm_port = sockvar->vm_port; + saddr.svm_cid = sockvar->vm_cid; +#endif + if (!sock) { + qtfs_err("Invalid client sock, which is null\n"); + return -EINVAL; + } + + ret = sock->ops->connect(sock, (struct sockaddr *)&saddr, sizeof(saddr), 0); + if (ret < 0) { + return ret; + } +#ifdef QTFS_TEST_MODE + QTSOCK_SET_KEEPX(sock, 5); +#else + sock_set_keepalive(sock->sk); +#endif + + qtfs_sock_recvtimeo_set(sockvar->client_sock, QTFS_SOCK_RCVTIMEO, 0); + return 0; +} +//client侧type用不上 +static int qtfs_conn_sock_init(void *connvar, qtfs_conn_type_e type) +{ + struct qtfs_sock_var_s *sockvar = (struct qtfs_sock_var_s *)connvar; + struct socket *sock; + int ret; + +#ifdef QTFS_TEST_MODE + ret = sock_create_kern(&init_net, AF_INET, SOCK_STREAM, 0, &sock); +#else + ret = sock_create_kern(&init_net, AF_VSOCK, SOCK_STREAM, 0, &sock); +#endif + if (ret) { + qtfs_err("qtfs sock client init create sock failed.\n"); + return -EFAULT; + } +#ifdef QTFS_TEST_MODE + QTSOCK_SET_KEEPX(sock, 5); +#else + sock_set_keepalive(sock->sk); +#endif + sockvar->client_sock = sock; + + return 0; +} +#endif + +static int qtfs_conn_sock_recv(void *connvar, void *buf, size_t len, bool block) +{ + struct qtfs_sock_var_s *sockvar = (struct qtfs_sock_var_s *)connvar; + struct kvec v; + struct msghdr msg_recv; + memset(&msg_recv, 0, sizeof(msg_recv)); + v.iov_base = buf; + v.iov_len = len; + + return kernel_recvmsg(sockvar->client_sock, &msg_recv, &v, 1, + len, (block == true) ? MSG_WAITALL : MSG_DONTWAIT); +} + +static int qtfs_conn_sock_recv_iter(void *connvar, struct iov_iter *iov, bool block) +{ + struct qtfs_sock_var_s *sockvar = (struct qtfs_sock_var_s *)connvar; + struct msghdr msg_recv; + if (iov == NULL) { + qtfs_err("sock recv iter is invalid"); + return -EINVAL; + } + memset(&msg_recv, 0, sizeof(msg_recv)); + msg_recv.msg_iter = *iov; + return sock_recvmsg(sockvar->client_sock, &msg_recv, (block == true) ? MSG_WAITALL : MSG_DONTWAIT); +} + +static int qtfs_conn_sock_send_iter(void *connvar, struct iov_iter *iov) +{ + struct qtfs_sock_var_s *sockvar = (struct qtfs_sock_var_s *)connvar; + struct msghdr msg_send; + if (iov == NULL) { + qtfs_err("sock send iter is invalid"); + return -EINVAL; + } + memset(&msg_send, 0, sizeof(msg_send)); + msg_send.msg_iter = *iov; + return sock_sendmsg(sockvar->client_sock, &msg_send); +} + +static int qtfs_conn_sock_send(void *connvar, void *buf, size_t len) +{ + struct qtfs_sock_var_s *sockvar = (struct qtfs_sock_var_s *)connvar; + struct kvec v; + int ret; + struct msghdr msg_send; + memset(&msg_send, 0, sizeof(struct msghdr)); + + v.iov_base = buf; + v.iov_len = len; + + ret = kernel_sendmsg(sockvar->client_sock, &msg_send, &v, 1, len); + if (ret < 0) { + qtfs_err("qtfs sock send error, ret:%d.\n", ret); + } + return ret; +} + +static void qtfs_conn_sock_fini(void *connvar, qtfs_conn_type_e type) +{ + struct qtfs_sock_var_s *sockvar = (struct qtfs_sock_var_s *)connvar; + if (sockvar->client_sock == NULL) { + qtfs_err("qtfs client sock is NULL during sock_fini"); + } + + if (sockvar->client_sock != NULL) { + qtfs_err("qtfs conn sock finish."); + sock_release(sockvar->client_sock); + sockvar->client_sock = NULL; + } + +#ifdef QTFS_SERVER + if (type < QTFS_CONN_TYPE_INV) { + mutex_lock(&qtfs_server_main_sock[type].lock); + if (qtfs_server_main_sock[type].sock != NULL) { + sock_release(qtfs_server_main_sock[type].sock); + qtfs_server_main_sock[type].sock = NULL; + } + mutex_unlock(&qtfs_server_main_sock[type].lock); + } +#endif + return; +} + +static bool qtfs_conn_sock_connected(void *connvar) +{ + struct qtfs_sock_var_s *sockvar = (struct qtfs_sock_var_s *)connvar; + struct socket *sock = sockvar->client_sock; + __u8 tcpi_state; + if (sock == NULL) + return false; + tcpi_state = inet_sk_state_load(sock->sk); + if (tcpi_state == TCP_ESTABLISHED) + return true; + qtfs_warn("qtfs tcpi state:%u(define:TCP_ESTABLISHED=1 is connected) disconnect!", tcpi_state); + + return false; +} +#ifdef QTFS_CLIENT +void qtfs_sock_drop_recv_buf(void *connvar) +{ +#define TMP_STACK_LEN 64 + struct qtfs_sock_var_s *sockvar = (struct qtfs_sock_var_s *)connvar; + int ret = 0; + char buf[TMP_STACK_LEN]; + struct kvec vec_recv; + struct msghdr msg_recv; + vec_recv.iov_base = buf; + vec_recv.iov_len = TMP_STACK_LEN; + do { + ret = kernel_recvmsg(sockvar->client_sock, &msg_recv, &vec_recv, 1, + vec_recv.iov_len, MSG_DONTWAIT); + if (ret > 0) { + qtfs_err("drop invalid data len:%d", ret); + } + } while (ret > 0); + return; +} +#endif + +#ifdef QTFS_SERVER +static bool qtfs_conn_sock_inited(void *connvar, qtfs_conn_type_e type) +{ + if (type >= QTFS_CONN_TYPE_INV) { + qtfs_err("invalid type:%u", type); + return false; + } + return qtfs_server_main_sock[type].sock != NULL; +} +#endif + +int qtfs_sock_param_init(void) +{ +#ifdef QTFS_SERVER + int i; + for (i = 0; i < QTFS_CONN_TYPE_INV; i++) { + qtfs_server_main_sock[i].sock = NULL; + mutex_init(&qtfs_server_main_sock[i].lock); + } +#endif + return 0; +} + +int qtfs_sock_param_fini(void) +{ +#ifdef QTFS_SERVER + int i; + for (i = 0; i < QTFS_CONN_TYPE_INV; i++) { + mutex_lock(&qtfs_server_main_sock[i].lock); + if (qtfs_server_main_sock[i].sock != NULL) { + sock_release(qtfs_server_main_sock[i].sock); + qtfs_server_main_sock[i].sock = NULL; + } + mutex_unlock(&qtfs_server_main_sock[i].lock); + } +#endif + return 0; +} + +struct qtfs_conn_ops_s qtfs_conn_sock_ops = { + .conn_var_init = qtfs_conn_var_init, + .conn_var_fini = qtfs_conn_var_fini, + .conn_msg_clear = qtfs_conn_msg_clear, + .get_conn_msg_buf = qtfs_conn_msg_buf, + .conn_init = qtfs_conn_sock_init, + .conn_fini = qtfs_conn_sock_fini, + .conn_send = qtfs_conn_sock_send, + .conn_recv = qtfs_conn_sock_recv, + .conn_send_iter = qtfs_conn_sock_send_iter, + .conn_recv_iter = qtfs_conn_sock_recv_iter, +#ifdef QTFS_SERVER + .conn_new_connection = qtfs_conn_sock_server_accept, + .conn_inited = qtfs_conn_sock_inited, +#endif +#ifdef QTFS_CLIENT + .conn_new_connection = qtfs_conn_sock_client_connect, + .conn_recv_buff_drop = qtfs_sock_drop_recv_buf, +#endif + .conn_connected = qtfs_conn_sock_connected, +}; + +int qtfs_sock_pvar_init(void *connvar, struct qtfs_conn_ops_s **conn_ops, qtfs_conn_type_e type) +{ + struct qtfs_sock_var_s *sockvar = (struct qtfs_sock_var_s *)connvar; + + if (type >= QTFS_CONN_TYPE_INV) { + qtfs_err("invalid type:%u", type); + return -1; + } + +#ifdef QTFS_TEST_MODE + // fill conn_pvar struct here + strlcpy(sockvar->addr, qtfs_server_ip, sizeof(sockvar->addr)); + sockvar->port = qtfs_server_port + type; +#else + // vsock + sockvar->vm_cid = qtfs_server_vsock_cid; + sockvar->vm_port = qtfs_server_vsock_port + type; +#endif + *conn_ops = &qtfs_conn_sock_ops; + return 0; +} + +static int qtfs_sock_parse_param(void) +{ + // parse conn specific params here + return 0; +} + +struct qtfs_pvar_ops_s qtfs_conn_sock_pvar_ops = { + .parse_param = qtfs_sock_parse_param, + .param_init = qtfs_sock_param_init, + .param_fini = qtfs_sock_param_fini, + .pvar_init = qtfs_sock_pvar_init +}; + +#ifdef QTFS_TEST_MODE +module_param_string(qtfs_server_ip, qtfs_server_ip, sizeof(qtfs_server_ip), 0600); +MODULE_PARM_DESC(qtfs_server_ip, "qtfs server ip"); +module_param(qtfs_server_port, int, 0600); +#else +module_param(qtfs_server_vsock_port, uint, 0600); +module_param(qtfs_server_vsock_cid, uint, 0600); +#endif \ No newline at end of file diff --git a/qtfs/qtfs_common/symbol_wrapper.c b/qtfs/qtfs_common/symbol_wrapper.c new file mode 100644 index 0000000..c7b7122 --- /dev/null +++ b/qtfs/qtfs_common/symbol_wrapper.c @@ -0,0 +1,296 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "conn.h" +#include "symbol_wrapper.h" + +unsigned long *symbols_origin[SYMBOL_MAX_NUM]; + +static struct kprobe kp = { + .symbol_name = "kallsyms_lookup_name" +}; +struct pt_regs; + +#ifdef __x86_64__ +#define WRAPPER_ARGS_TO_REGS1(regs) regs->di = (unsigned long)x1; + +#define WRAPPER_ARGS_TO_REGS2(regs) \ + WRAPPER_ARGS_TO_REGS1(regs)\ + regs->si = (unsigned long)x2; + +#define WRAPPER_ARGS_TO_REGS3(regs) \ + WRAPPER_ARGS_TO_REGS2(regs)\ + regs->dx = (unsigned long)x3; + +#define WRAPPER_ARGS_TO_REGS4(regs) \ + WRAPPER_ARGS_TO_REGS3(regs)\ + regs->r10 = (unsigned long)x4; + +#define WRAPPER_ARGS_TO_REGS5(regs) \ + WRAPPER_ARGS_TO_REGS4(regs)\ + regs->r8 = (unsigned long)x5; + +#define WRAPPER_ARGS_TO_REGS6(regs) \ + WRAPPER_ARGS_TO_REGS5(regs)\ + regs->r9 = (unsigned long)x6; +#endif +#ifdef __aarch64__ +void (*update_mapping_prot)(phys_addr_t phys, unsigned long virt, phys_addr_t size, pgprot_t prot); +unsigned long start_rodata, end_rodata; + +// symbols not finded in sys call table +enum qtfs_sym_a64 { + A64_NR_UNLINK = 0, + A64_NR_RMDIR, + A64_NR_EPOLL_WAIT, + A64_NR_MAX, +}; +unsigned long *symbols_a64[A64_NR_MAX]; + +#define WRAPPER_ARGS_TO_REGS1(regs) regs->regs[0] = (unsigned long)x1; + +#define WRAPPER_ARGS_TO_REGS2(regs)\ + WRAPPER_ARGS_TO_REGS1(regs)\ + regs->regs[1] = (unsigned long)x2; + +#define WRAPPER_ARGS_TO_REGS3(regs)\ + WRAPPER_ARGS_TO_REGS2(regs)\ + regs->regs[2] = (unsigned long)x3; + +#define WRAPPER_ARGS_TO_REGS4(regs)\ + WRAPPER_ARGS_TO_REGS3(regs)\ + regs->regs[3] = (unsigned long)x4; + +#define WRAPPER_ARGS_TO_REGS5(regs)\ + WRAPPER_ARGS_TO_REGS4(regs)\ + regs->regs[4] = (unsigned long)x5; + +#define WRAPPER_ARGS_TO_REGS6(regs)\ + WRAPPER_ARGS_TO_REGS5(regs)\ + regs->regs[5] = (unsigned long)x6; +#endif + +#define KSYMS(sym, type) \ + qtfs_kern_syms.sym = (type)qtfs_kallsyms_lookup_name(#sym); + +#define KSYMS_NULL_RETURN(sym)\ + if (sym == NULL) {\ + qtfs_err("symbol:%s not finded", #sym);\ + return -1;\ + } + +struct qtfs_kallsyms qtfs_kern_syms; +kallsyms_lookup_name_t qtfs_kallsyms_lookup_name; +int qtfs_kallsyms_hack_init(void) +{ + int ret = register_kprobe(&kp); + if (ret < 0) { + qtfs_err("register kprobe failed, Please confirm whether kprobe is enabled, ret:%d", ret); + return -1; + } + qtfs_kallsyms_lookup_name = (kallsyms_lookup_name_t) kp.addr; + unregister_kprobe(&kp); + if (qtfs_kallsyms_lookup_name == NULL) { + qtfs_err("get kallsyms function by kprobe failed."); + return -1; + } + + KSYMS(sys_call_table, unsigned long **); + KSYMS_NULL_RETURN(qtfs_kern_syms.sys_call_table); + + KSYMS(d_absolute_path, char *(*)(const struct path *, char *, int)); + KSYMS_NULL_RETURN(qtfs_kern_syms.d_absolute_path); + KSYMS(find_get_task_by_vpid, struct task_struct *(*)(pid_t nr)); + KSYMS_NULL_RETURN(qtfs_kern_syms.find_get_task_by_vpid); + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0)) + KSYMS(__close_fd, int (*)(struct files_struct *, int)); + KSYMS_NULL_RETURN(qtfs_kern_syms.__close_fd); +#endif + +#ifdef __aarch64__ + update_mapping_prot = (void *)qtfs_kallsyms_lookup_name("update_mapping_prot"); + start_rodata = (unsigned long)qtfs_kallsyms_lookup_name("__start_rodata"); + end_rodata = (unsigned long)qtfs_kallsyms_lookup_name("__end_rodata"); + if (update_mapping_prot == NULL || start_rodata == NULL || end_rodata == NULL) { + qtfs_err("failed to init memory protect handler"); + return -1; + } + +#pragma GCC diagnostic ignored "-Wint-conversion" + symbols_a64[A64_NR_UNLINK] = (unsigned long)qtfs_kallsyms_lookup_name("__arm64_sys_unlink"); + KSYMS_NULL_RETURN(symbols_a64[A64_NR_UNLINK]); + symbols_a64[A64_NR_RMDIR] = (unsigned long)qtfs_kallsyms_lookup_name("__arm64_sys_rmdir"); + KSYMS_NULL_RETURN(symbols_a64[A64_NR_RMDIR]); + symbols_a64[A64_NR_EPOLL_WAIT] = (unsigned long)qtfs_kallsyms_lookup_name("__arm64_sys_epoll_wait"); + KSYMS_NULL_RETURN(symbols_a64[A64_NR_EPOLL_WAIT]); +#pragma GCC diagnostic pop + qtfs_info("finded __arm64_sys_unlink"); + qtfs_info("finded __arm64_sys_rmdir"); + qtfs_info("finded __arm64_sys_epoll_wait"); +#endif + + return 0; +} + +__SYSCALL_DEFINEx(3, _qtfs_connect, int, fd, struct sockaddr __user *, uservaddr, int, addrlen) +{ + return qtfs_uds_remote_connect_user(fd, uservaddr, addrlen); +} + +static atomic_t replace_available = ATOMIC_INIT(1); + +int qtfs_syscall_replace_start(void) +{ + if (!atomic_dec_and_test(&replace_available)) { + atomic_inc(&replace_available); + return -EBUSY; + } + + symbols_origin[SYMBOL_SYSCALL_CONNECT] = qtfs_kern_syms.sys_call_table[__NR_connect]; +#ifdef __x86_64__ + make_rw((unsigned long)qtfs_kern_syms.sys_call_table); + qtfs_kern_syms.sys_call_table[__NR_connect] = (unsigned long *)__x64_sys_qtfs_connect; + make_ro((unsigned long)qtfs_kern_syms.sys_call_table); +#endif + +#ifdef __aarch64__ + update_mapping_prot(__pa_symbol(start_rodata), (unsigned long)start_rodata, section_size, PAGE_KERNEL); + qtfs_kern_syms.sys_call_table[__NR_connect] = (unsigned long *)__arm64_sys_qtfs_connect; + update_mapping_prot(__pa_symbol(start_rodata), (unsigned long)start_rodata, section_size, PAGE_KERNEL_RO); +#endif + return 0; +} + +void qtfs_syscall_replace_stop(void) +{ +#ifdef __x86_64__ + make_rw((unsigned long)qtfs_kern_syms.sys_call_table); + qtfs_kern_syms.sys_call_table[__NR_connect] = (void *)qtfs_kallsyms_lookup_name("__x64_sys_connect"); + make_ro((unsigned long)qtfs_kern_syms.sys_call_table); +#endif + +#ifdef __aarch64__ + update_mapping_prot(__pa_symbol(start_rodata), (unsigned long)start_rodata, section_size, PAGE_KERNEL); + qtfs_kern_syms.sys_call_table[__NR_connect] = (void *)qtfs_kallsyms_lookup_name("__arm64_sys_connect"); + update_mapping_prot(__pa_symbol(start_rodata), (unsigned long)start_rodata, section_size, PAGE_KERNEL_RO); +#endif + atomic_inc(&replace_available); +} + +#define SYSCALL_TYPE(type) (type (*)(const struct pt_regs *)) + +#define WRAPPER_DEFINE_BYORIGIN(nargs, ret, func, nr)\ + noinline ret func\ + {\ + struct pt_regs _regs;\ + struct pt_regs *regs = &_regs;\ + ret retval;\ + WRAPPER_ARGS_TO_REGS##nargs(regs);\ + retval = (SYSCALL_TYPE(ret) symbols_origin[nr])(regs);\ + return retval;\ + } + +#ifdef QTFS_CLIENT +WRAPPER_DEFINE_BYORIGIN(2, long, qtfs_syscall_umount(char __user *x1, int x2), SYMBOL_SYSCALL_UMOUNT); +WRAPPER_DEFINE_BYORIGIN(5, long, qtfs_syscall_mount(char __user *x1, char __user *x2, + char __user *x3, unsigned long x4, void __user *x5), SYMBOL_SYSCALL_MOUNT); +WRAPPER_DEFINE_BYORIGIN(4, long, qtfs_syscall_epoll_ctl(int x1, int x2, int x3, + struct epoll_event __user *x4), SYMBOL_SYSCALL_EPOLL_CTL); +#endif + +#define WRAPPER_DEFINE(nargs, ret, func, nr)\ + noinline ret func\ + {\ + struct pt_regs _regs;\ + struct pt_regs *regs = &_regs;\ + ret retval;\ + WRAPPER_ARGS_TO_REGS##nargs(regs);\ + retval = (SYSCALL_TYPE(ret) qtfs_kern_syms.sys_call_table[nr])(regs);\ + return retval;\ + } + +//common syscall wrapper +WRAPPER_DEFINE_BYORIGIN(3, long, qtfs_syscall_connect(int x1, struct sockaddr __user *x2, + int x3), SYMBOL_SYSCALL_CONNECT); + +// only use in server syscall wrapper +#ifdef QTFS_SERVER +WRAPPER_DEFINE(2, long, qtfs_syscall_umount(char __user *x1, int x2), __NR_umount2); +WRAPPER_DEFINE(5, long, qtfs_syscall_mount(char __user *x1, char __user *x2, + char __user *x3, unsigned long x4, void __user *x5), __NR_mount); +WRAPPER_DEFINE(4, long, qtfs_syscall_epoll_ctl(int x1, int x2, int x3, + struct epoll_event __user *x4), __NR_epoll_ctl); +WRAPPER_DEFINE(4, int, qtfs_syscall_readlinkat(int x1, const char __user *x2, + char __user *x3, int x4), __NR_readlinkat); +WRAPPER_DEFINE(5, int, qtfs_syscall_renameat2(int x1, const char __user *x2, + int x3, const char __user *x4, unsigned int x5), __NR_renameat2); +WRAPPER_DEFINE(3, long, qtfs_syscall_mkdirat(int x1, const char __user *x2, + umode_t x3), __NR_mkdirat); +WRAPPER_DEFINE(2, int, qtfs_syscall_statfs(const char __user *x1, + struct statfs __user *x2), __NR_statfs); +WRAPPER_DEFINE(4, int, qtfs_syscall_openat(int x1, const char __user *x2, int x3, + umode_t x4), __NR_openat); +WRAPPER_DEFINE(5, int, qtfs_syscall_linkat(int x1, const char __user *x2, + int x3, const char __user *x4, int x5), __NR_linkat); +WRAPPER_DEFINE(4, long, qtfs_syscall_mknodat(int x1, const char __user *x2, umode_t x3, + unsigned int x4), __NR_mknodat); +WRAPPER_DEFINE(3, off_t, qtfs_syscall_lseek(unsigned int x1, off_t x2, + unsigned int x3), __NR_lseek); +WRAPPER_DEFINE(2, long, qtfs_syscall_kill(pid_t x1, int x2), __NR_kill); +WRAPPER_DEFINE(3, long, qtfs_syscall_sched_getaffinity(pid_t x1, unsigned int x2, + unsigned long __user *x3), __NR_sched_getaffinity); +WRAPPER_DEFINE(3, long, qtfs_syscall_sched_setaffinity(pid_t x1, unsigned int x2, + unsigned long __user *x3), __NR_sched_setaffinity); +WRAPPER_DEFINE(3, long, qtfs_syscall_write(unsigned int x1, const char __user *x2, + size_t x3), __NR_write); +WRAPPER_DEFINE(3, long, qtfs_syscall_read(unsigned int x1, char __user *x2, + size_t x3), __NR_read); +WRAPPER_DEFINE(3, long, qtfs_syscall_ioctl(unsigned int x1, unsigned int x2, + unsigned long x3), __NR_ioctl); + +#ifdef __aarch64__ +#define WRAPPER_DEFINE_A64(nargs, ret, func, nr)\ + noinline ret func\ + {\ + struct pt_regs _regs;\ + struct pt_regs *regs = &_regs;\ + ret retval;\ + WRAPPER_ARGS_TO_REGS##nargs(regs);\ + retval = (SYSCALL_TYPE(ret) symbols_a64[nr])(regs);\ + return retval;\ + } + +// ARM64 syscall not finded in sys_call_table is defined here +WRAPPER_DEFINE_A64(1, long, qtfs_syscall_unlink(const char __user *x1), A64_NR_UNLINK); +WRAPPER_DEFINE_A64(4, int, qtfs_syscall_epoll_wait(int x1, struct epoll_event __user *x2, + int x3, int x4), A64_NR_EPOLL_WAIT); +WRAPPER_DEFINE_A64(1, long, qtfs_syscall_rmdir(const char __user *x1), A64_NR_RMDIR); +#else +WRAPPER_DEFINE(1, long, qtfs_syscall_unlink(const char __user *x1), __NR_unlink); +WRAPPER_DEFINE(4, int, qtfs_syscall_epoll_wait(int x1, struct epoll_event __user *x2, + int x3, int x4), __NR_epoll_wait); +WRAPPER_DEFINE(1, long, qtfs_syscall_rmdir(const char __user *x1), __NR_rmdir); +#endif +#endif + + diff --git a/qtfs/qtfs_common/user_engine.c b/qtfs/qtfs_common/user_engine.c new file mode 100644 index 0000000..3337c34 --- /dev/null +++ b/qtfs/qtfs_common/user_engine.c @@ -0,0 +1,521 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * qtfs licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Liqiang + * Create: 2023-03-20 + * Description: + *******************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "comm.h" +#include "ipc/uds_main.h" + +char wl_type_str[QTFS_WHITELIST_MAX][16] = { + "Open", + "Write", + "Read", + "Readdir", + "Mkdir", + "Rmdir", + "Create", + "Unlink", + "Rename", + "Setattr", + "Setxattr", + "Mount", + "Kill", + "Udsconnect" + }; + +#define engine_out(info, ...) \ + do {\ + printf("[Engine::%s:%3d]"info"\n", __func__, __LINE__, ##__VA_ARGS__);\ + } while (0); + +#define engine_out2(info, ...) \ + do {\ + printf("[Engine::%s:%3d]"info, __func__, __LINE__, ##__VA_ARGS__);\ + } while (0); + +#define engine_err(info, ...) \ + do {\ + printf("[ERROR:Engine::%s:%3d]"info"\n", __func__, __LINE__, ##__VA_ARGS__);\ + } while (0); + +#define WHITELIST_FILE "/etc/qtfs/whitelist" + +struct engine_arg { + int psize; + int fd; + int thread_idx; +}; + +#define QTFS_USERP_SIZE QTFS_USERP_MAXSIZE +#define QTFS_SERVER_FILE "/dev/qtfs_server" +#define ENGINE_LOCK_ADDR "/var/run/qtfs/engine.lock" +#define ENGINE_LOCK_FILE_DIR "/var/run/qtfs/" + +static volatile sig_atomic_t sig_int_flag = 0; + +static int engine_env_prepare() +{ + DIR *dir; + if (access(ENGINE_LOCK_ADDR, 0) == 0) + return 0; + if ((dir = opendir(ENGINE_LOCK_FILE_DIR)) == NULL) { + if (mkdir(ENGINE_LOCK_FILE_DIR, 0600) < 0) { + engine_err("mkdir %s failed.", ENGINE_LOCK_FILE_DIR); + return -1; + } + } else { + closedir(dir); + } + return 0; +} + +int engine_socket_lock(void) +{ + int lock_fd = open(ENGINE_LOCK_ADDR, O_RDONLY | O_CREAT, 0600); + if (lock_fd == -1) + return -EINVAL; + + return flock(lock_fd, LOCK_EX | LOCK_NB); +} + +int qtfs_fd; +int engine_run = 1; +static void qtfs_engine_userp_free(struct qtfs_server_userp_s *userp, int thread_nums) +{ + for (int i = 0; i < thread_nums; i++) { + if (userp[i].userp != NULL) + free(userp[i].userp); + if (userp[i].userp2 != NULL) + free(userp[i].userp2); + } + free(userp); + return; +} + +static struct qtfs_server_userp_s *qtfs_engine_thread_init(int fd, int thread_nums, int psize) +{ + struct qtfs_server_userp_s *userp; + userp = (struct qtfs_server_userp_s *)calloc(thread_nums, sizeof(struct qtfs_server_userp_s)); + if (userp == NULL) { + engine_out("engine thread init malloc failed."); + return NULL; + } + for (int i = 0; i < thread_nums; i++) { + userp[i].size = psize; + userp[i].userp = (void *)memalign(sysconf(_SC_PAGESIZE), psize); + if (userp[i].userp == NULL) { + engine_out("engine userp malloc failed."); + goto rollback; + } + userp[i].userp2 = (void *)memalign(sysconf(_SC_PAGESIZE), psize); + if (userp[i].userp2 == NULL) { + engine_out("engine userp2 malloc failed."); + goto rollback; + } + } + struct qtfs_thread_init_s init_userp; + int ret; + init_userp.thread_nums = thread_nums; + init_userp.userp = userp; + ret = ioctl(fd, QTFS_IOCTL_THREAD_INIT, (unsigned long)&init_userp); + if (ret != QTOK) { + engine_err("Engine thread init failed reason:%s", + (ret == EADDRINUSE) ? "Address already in use" : "userp init failed."); + goto rollback; + } + return userp; +rollback: + qtfs_engine_userp_free(userp, thread_nums); + return NULL; +} + +static void *qtfs_engine_kthread(void *arg) +{ + struct engine_arg *parg = (struct engine_arg *)arg; + int psize = parg->psize; + long ret; + + while (engine_run) { + ret = ioctl(parg->fd, QTFS_IOCTL_THREAD_RUN, 0); + if (ret == QTEXIT) { + engine_out("qtfs server thread:%d exit.", parg->thread_idx); + break; + } + if (sig_int_flag == 1) { + engine_out("qtfs engine recv SIGINT."); + + if (qtfs_fd < 0) { + engine_err("qtfs engine signal int file:%s open failed, fd:%d.", QTFS_SERVER_FILE, qtfs_fd); + continue; + } + ret = ioctl(qtfs_fd, QTFS_IOCTL_EXIT, 0); + engine_out("qtfs engine send QTFS_IOCTL_EXIT to kernel, get return value:%ld.", ret); + engine_run = 0; + break; + } + if (ret != QTOK) { + usleep(1000); + } + } + +end: + engine_out("qtfs user engine over."); + return NULL; +} + +static void qtfs_signal_int(int signum) +{ + sig_int_flag = 1; + return; +} + +static void *qtfs_engine_epoll_thread(void *data) +{ + struct engine_arg *arg = (struct engine_arg *)data; + int fd = arg->fd; + long ret; + + engine_out("qtfs server epoll thread run."); + + do { + ret = ioctl(fd, QTFS_IOCTL_EPOLL_THREAD_INIT, NULL); + if (ret == QTEXIT) { + engine_out("qtfs server epoll thread init exit."); + goto end; + } + } while (ret != 0 && engine_run); + while (engine_run) { + ret = ioctl(fd, QTFS_IOCTL_EPOLL_THREAD_RUN, NULL); + if (ret == QTEXIT) { + engine_out("qtfs server epoll thread exit."); + break; + } + if (ret != QTOK) { + usleep(1000); + } + } +end: + engine_out("qtfs server epoll thread exit, ret:%ld.", ret); + + return NULL; +} + +int qtfs_epoll_init(int fd) +{ + int epfd = epoll_create1(0); + if (epfd < 0) { + engine_err("epoll create error, ret:%d.", epfd); + return -1; + } + + struct qtfs_server_epoll_s ep; + struct epoll_event *evts; + evts = calloc(QTFS_MAX_EPEVENTS_NUM, sizeof(struct epoll_event)); + if (evts == NULL) { + engine_err("calloc events failed."); + close(epfd); + return -1; + } + engine_out("qtfs engine set epoll arg, fd:%d event nums:%d.", epfd, QTFS_MAX_EPEVENTS_NUM); + ep.epfd = epfd; + ep.event_nums = QTFS_MAX_EPEVENTS_NUM; + ep.events = evts; + int ret = ioctl(fd, QTFS_IOCTL_EPFDSET, &ep); + if (ret != 0) { + engine_err("failed to set epoll fd, ret:%d.", ret); + close(epfd); + return -1; + } + + return epfd; +} + +static void qtfs_whitelist_free_items(char **items, gsize count) +{ + for (gsize j = 0; j < count; j++) { + if (items[j]) + free(items[j]); + } + if (items) + free(items); + return; +} + +static int qtfs_whitelist_transfer(int fd, GKeyFile *config, int type) +{ + gsize i, wl_count; + int ret; + char **items; + struct qtfs_wl_item head; + items = g_key_file_get_string_list(config, wl_type_str[type], "Path", &wl_count, NULL); + if (wl_count <= 0) { + engine_err("Can't find whitelist item %s", wl_type_str[type]); + return -1; + } + if (wl_count > QTFS_WL_MAX_NUM) + wl_count = QTFS_WL_MAX_NUM; + + head.type = type; + head.index = 0; // not use in add + for(i = 0; i < wl_count; i++){ + head.len = strlen(items[i]); + if (head.len >= QTFS_PATH_MAX) { + engine_err("Whitelist type:%s invalid path:%s is too long(> %d)", wl_type_str[type], items[i], QTFS_PATH_MAX - 1); + continue; + } + head.path = items[i]; + ret = ioctl(fd, QTFS_IOCTL_WL_ADD, &head); + if (ret == QTERROR) { + engine_err("Failed to add whitelist:%s type:%d, engine start failed.", items[i], type); + goto end; + } + } + engine_out("Successed to add white list items type:%s count:%d", wl_type_str[type], wl_count); + qtfs_whitelist_free_items(items, wl_count); + return 0; + +end: + qtfs_whitelist_free_items(items, wl_count); + return -1; +} + +int qtfs_whitelist_init(int fd) +{ + int mount_whitelist = 0; + int ret, i; + GKeyFile *config = g_key_file_new(); + g_key_file_load_from_file(config, WHITELIST_FILE, G_KEY_FILE_KEEP_COMMENTS|G_KEY_FILE_KEEP_TRANSLATIONS, NULL); + for (i = 0; i < QTFS_WHITELIST_MAX; i++) { + ret = qtfs_whitelist_transfer(fd, config, i); + if (ret != 0) { + engine_err("failed to set whitelist item %s, get error:%d", wl_type_str[i], ret); + // failure of one whitelist type should not stop others. + continue; + } + if (i == QTFS_WHITELIST_MOUNT) + mount_whitelist = 1; + } + g_key_file_free(config); + // if wl file not exist or mount not set, result is mount_whitelist == 0, stop the engine + if (mount_whitelist == 0) { + engine_err("Please create whitelist file and add white list items."); + engine_err("At least one [Mount] whitelist is required for the qtfs to work normally."); + return -1; + } + + return 0; +} + +#ifdef UDS_TEST_MODE +static int qtfs_engine_check_port(unsigned short port, char *ip) +#else +static int qtfs_engine_check_port(unsigned short port, char *scid) +#endif +{ +#ifdef UDS_TEST_MODE + struct sockaddr_in sin; + if (inet_pton(AF_INET, ip, &sin.sin_addr) != 1) { + engine_err("%s inet_pton error.", ip); + return -1; + } + int sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) { + engine_err("socket error, fd:%d", sockfd); + return -1; + } + bzero(&sin, sizeof(sin)); + sin.sin_family = AF_INET; + sin.sin_port = htons(port); + if (bind(sockfd, (struct sockaddr *)&sin, sizeof(struct sockaddr)) < 0) { + engine_err("ip:%s port:%u bind failed, errno:%d.", ip, port, errno); + close(sockfd); + return -1; + } + close(sockfd); +#else +#define DEC 10 + long cid = strtol(scid, NULL, DEC); // base 10 + if (errno == ERANGE) { + engine_err("The cid value out of range\n"); + return -1; + } + + if (cid != -1) { + if (cid < VMADDR_CID_HOST || cid > 0xFFFFFFFF) { + engine_err("The cid value[%ld] was invalid\n", cid); + return -1; + } + } + + struct sockaddr_vm saddr; + memset(&saddr, 0, sizeof(saddr)); + saddr.svm_family = AF_VSOCK; + saddr.svm_port = port; + saddr.svm_cid = cid; + + int sock_fd = socket(saddr.svm_family, SOCK_STREAM, 0); + if (bind(sock_fd, (struct sockaddr *)&saddr, sizeof(saddr)) < 0) { + engine_err("cid:%ld port:%u bind failed, errno:%d.", cid, port, errno); + close(sock_fd); + return -1; + } + close(sock_fd); +#endif + return 0; +} + +#define IS_NUMBER(c) (c >= '0' && c <= '9') +static int qtfs_engine_env_check(char *argv[]) +{ + struct qtinfo diag; + int ret; + int fd; + for (int i = 0; i < strlen(argv[1]); i ++) { + if (!IS_NUMBER(argv[1][i])) { + engine_err("invalid thread number :%s", argv[1]); + return -1; + } + } + + if (qtfs_engine_check_port(atoi(argv[4]), argv[3]) < 0) + goto err; + + return 0; +err: + return -1; +} + +#define QTFS_ENGINE_FD_LIMIT 65536 +static void engine_rlimit() +{ + struct rlimit lim; + + getrlimit(RLIMIT_NOFILE, &lim); + if (lim.rlim_cur >= QTFS_ENGINE_FD_LIMIT) + return; + engine_out("engine fd cur limit:%d, change to:%d", lim.rlim_cur, QTFS_ENGINE_FD_LIMIT); + lim.rlim_cur = QTFS_ENGINE_FD_LIMIT; + setrlimit(RLIMIT_NOFILE, &lim); + return; +} + +int main(int argc, char *argv[]) +{ + int ret = 0; + if (argc != 7) { + engine_out("Usage: %s .", argv[0]); + engine_out(" Example: %s 16 1 192.168.10.10 12121 192.168.10.11 12121.", argv[0]); + return -1; + } + if (engine_env_prepare() != 0 || engine_socket_lock() < 0) { + engine_err("Engine is running."); + return -1; + } + if (qtfs_engine_env_check(argv) < 0) { + engine_err("Environment check failed, engine exit."); + return -1; + } + engine_rlimit(); + int thread_nums = atoi(argv[1]); + int fd = open(QTFS_SERVER_FILE, O_RDONLY); + if (fd < 0) { + engine_err("qtfs server file:%s open failed, fd:%d.", QTFS_SERVER_FILE, fd); + return -1; + } + qtfs_fd = fd; + // init epoll + int epfd = qtfs_epoll_init(fd); + if (epfd < 0) { + close(fd); + return -1; + } + ret = ioctl(fd, QTFS_IOCTL_EXIT, 1); + if (ret == QTERROR) { + goto end; + } + ret = qtfs_whitelist_init(fd); + if (ret) + goto end; + + umask(0); + + pthread_t texec[QTFS_MAX_THREADS]; + pthread_t tepoll; + if (thread_nums < 0 || thread_nums > QTFS_MAX_THREADS) { + engine_err("qtfs engine parm invalid, thread_nums:%d(must <= %d).", + thread_nums, QTFS_MAX_THREADS); + ret = -1; + goto end; + } + signal(SIGINT, qtfs_signal_int); + signal(SIGTERM, qtfs_signal_int); + + struct qtfs_server_userp_s *userp = qtfs_engine_thread_init(fd, thread_nums, QTFS_USERP_SIZE); + if (userp == NULL) { + engine_out("qtfs engine userp init failed."); + ret = -1; + goto end; + } + struct engine_arg arg[QTFS_MAX_THREADS]; + for (int i = 0; i < thread_nums; i++) { + arg[i].psize = QTFS_USERP_SIZE; + arg[i].fd = fd; + arg[i].thread_idx = i; + (void)pthread_create(&texec[i], NULL, qtfs_engine_kthread, &arg[i]); + } + (void)pthread_create(&tepoll, NULL, qtfs_engine_epoll_thread, &arg[0]); + // 必须放在这个位置，uds main里面最终也有join + if (uds_proxy_main(6, &argv[1]) != 0) { + engine_out("uds proxy start failed."); + ret = -1; + goto engine_free; + } + for (int i = 0; i < thread_nums; i++) { + pthread_join(texec[i], NULL); + engine_out("qtfs engine join thread %d.", i); + } + pthread_join(tepoll, NULL); +engine_free: + qtfs_engine_userp_free(userp, thread_nums); + engine_out("qtfs engine join epoll thread."); +end: + (void)ioctl(fd, QTFS_IOCTL_EXIT, 0); + close(epfd); + close(fd); + engine_out("qtfs engine over."); + return ret; +} diff --git a/qtfs/qtfs_server/CMakeLists.txt b/qtfs/qtfs_server/CMakeLists.txt new file mode 100644 index 0000000..3cdb5f4 --- /dev/null +++ b/qtfs/qtfs_server/CMakeLists.txt @@ -0,0 +1,69 @@ +# Build Kernel modules +set(MODULE_NAME qtfs_server) + +## First, find the kernel build directory +execute_process( + COMMAND uname -r + OUTPUT_VARIABLE KERNEL_RELEASE + OUTPUT_STRIP_TRAILING_WHITESPACE) +set(KBUILD_DIR /lib/modules/${KERNEL_RELEASE}/build/) +find_file(KERNEL_MAKEFILE NAMES Makefile PATHS ${KBUILD_DIR} NO_DEFAULT_PATH) +if (NOT KERNEL_MAKEFILE) + message(FATAL_ERROR "There is no Makefile in ${KBUILD_DIR}!") +endif () + +## Second, gather the source files +set(COMM_SRC_FILES + ${QTFS_BASE_DIR}/qtfs_common/conn.c + ${QTFS_BASE_DIR}/qtfs_common/misc.c + ${QTFS_BASE_DIR}/qtfs_common/qtfs_check.c + ${QTFS_BASE_DIR}/qtfs_common/socket.c + ${QTFS_BASE_DIR}/qtfs_common/symbol_wrapper.c) + +set(QTFS_SERVER_SRC_FILES + ${QTFS_BASE_DIR}/qtfs_server/fsops.c + ${QTFS_BASE_DIR}/qtfs_server/qtfs-server.c) + +set(QTFS_SERVER_MODULE_SRC ${COMM_SRC_FILES} ${QTFS_SERVER_SRC_FILES}) + +## Third, make the src files accessible to the kernel Makefile +set(MODULE_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/) +set(QTFS_SERVER_KBUILD_FILE "obj-m := ${MODULE_NAME}.o") +foreach (file ${QTFS_SERVER_MODULE_SRC}) + file(RELATIVE_PATH file "${CMAKE_CURRENT_SOURCE_DIR}" "${file}") + configure_file(${file} ${file} COPYONLY) + string(REPLACE ".c" ".o" file_obj "${file}") + set(QTFS_SERVER_KBUILD_FILE "${QTFS_SERVER_KBUILD_FILE}\n${MODULE_NAME}-y += ${file_obj}") +endforeach () + +set(QTFS_SERVER_KBUILD_FILE "${QTFS_SERVER_KBUILD_FILE}\nccflags-y += -I${QTFS_BASE_DIR}/include") +set(QTFS_SERVER_KBUILD_FILE "${QTFS_SERVER_KBUILD_FILE}\nccflags-y += -I${QTFS_BASE_DIR}/ipc") +set(QTFS_SERVER_KBUILD_FILE "${QTFS_SERVER_KBUILD_FILE}\nccflags-y += -I${QTFS_BASE_DIR}/qtfs_server") +set(QTFS_SERVER_KBUILD_FILE "${QTFS_SERVER_KBUILD_FILE}\nccflags-y += -DQTFS_SERVER") + +if (DEFINED UDS_TEST_MODE OR DEFINED QTFS_TEST_MODE) + set(QTFS_SERVER_KBUILD_FILE "${QTFS_SERVER_KBUILD_FILE}\nccflags-y += -DQTFS_TEST_MODE") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DUDS_TEST_MODE") +endif () + +## Forth, generate a Kbuild file +file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/Kbuild ${QTFS_SERVER_KBUILD_FILE}) + +## Fifth, add a custom target to build the module +set(MODULE_CMD ${CMAKE_MAKE_PROGRAM} -C ${KBUILD_DIR} M=${CMAKE_CURRENT_BINARY_DIR}) +add_custom_command(OUTPUT ${MODULE_NAME}.ko + COMMAND ${MODULE_CMD} modules + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${QTFS_SERVER_MODULE_SRC} ${CMAKE_CURRENT_BINARY_DIR}/Kbuild + VERBATIM) +add_custom_target(${MODULE_NAME} DEPENDS ${MODULE_NAME}.ko) +add_custom_target(${MODULE_NAME}-clean COMMAND ${MODULE_CMD} clean) + +# This target helps parsing C files for IDEs like CLion +add_library(dummy-${MODULE_NAME} EXCLUDE_FROM_ALL ${QTFS_SERVER_MODULE_SRC}) +target_include_directories(dummy-${MODULE_NAME} PRIVATE + ${KBUILD_DIR}/include + ${QTFS_BASE_DIR}/include + ${QTFS_BASE_DIR}/ipc + ${QTFS_BASE_DIR}/qtfs_server) +set_target_properties(dummy-${MODULE_NAME} PROPERTIES DEPRECATION "DO NOT BUILD THIS TARGET") \ No newline at end of file diff --git a/qtfs/qtfs_server/License b/qtfs/qtfs_server/License new file mode 100644 index 0000000..3912109 --- /dev/null +++ b/qtfs/qtfs_server/License @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/qtfs/qtfs_server/Makefile b/qtfs/qtfs_server/Makefile new file mode 100644 index 0000000..08b70f5 --- /dev/null +++ b/qtfs/qtfs_server/Makefile @@ -0,0 +1,50 @@ +ifdef QTFS_TEST_MODE +ccflags-y += -I$(src)/../ -I$(src) -I$(src)/../ipc/ -I$(src)/../include/ -DQTFS_SERVER -DQTFS_TEST_MODE +CFLAGS += -DUDS_TEST_MODE +else +ccflags-y += -I$(src)/../ -I$(src) -I$(src)/../ipc/ -I$(src)/../include/ -DQTFS_SERVER +endif + +CFLAGS += -g -O2 +CFLAGS += -fstack-protector-strong +CFLAGS += -fPIE -pie -fPIC +CFLAGS += -D_FORTIFY_SOURCE=2 +LDFLAGS += -Wl,-z,now +LDFLAGS += -Wl,-z,noexecstack +LDFLAGS += -fPIE -pie + +KBUILD=/lib/modules/$(shell uname -r)/build/ +COMM=../qtfs_common/ +COMMO=$(COMM)/conn.o $(COMM)/misc.o $(COMM)/symbol_wrapper.o $(COMM)/socket.o $(COMM)/qtfs_check.o + +obj-m:=qtfs_server.o +qtfs_server-objs:=fsops.o qtfs-server.o $(COMMO) + +DEPGLIB=-lglib-2.0 -I../ -I../include/ -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include + +all: qtfs_server engine + +qtfs_server: + make -C $(KBUILD) M=$(PWD) modules + @test -z $(QTFS_TEST_MODE) || echo "Important risk warning: The test mode is turned on,\ + and qtfs will expose the network port, which will bring security risks and is only for\ + testing! If you do not understand the risks, please don't use or compile again without\ + QTFS_TEST_MODE." + +engine: uds_event.o uds_main.o user_engine.o + gcc $(LDFLAGS) -o engine $^ -lpthread $(DEPGLIB) -I../ -I../ipc/ -DQTFS_SERVER + +user_engine.o: + cc $(CFLAGS) -c -o user_engine.o ../qtfs_common/user_engine.c $(DEPGLIB) -I../ -DQTFS_SERVER + +uds_event.o: + cc $(CFLAGS) -c -o uds_event.o ../ipc/uds_event.c -DQTFS_SERVER $(DEPGLIB) + +uds_main.o: + cc $(CFLAGS) -c -o uds_main.o ../ipc/uds_main.c -DQTFS_SERVER $(DEPGLIB) + +clean: + make -C $(KBUILD) M=$(PWD) clean + rm -rf engine + rm -rf ../*.o + rm -rf $(COMMO) $(COMM).*.o.cmd diff --git a/qtfs/qtfs_server/fsops.c b/qtfs/qtfs_server/fsops.c new file mode 100644 index 0000000..5f3f779 --- /dev/null +++ b/qtfs/qtfs_server/fsops.c @@ -0,0 +1,1734 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) +#include +#include +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0)) +#include +#endif + +#include "conn.h" +#include "qtfs-server.h" +#include "req.h" +#include "log.h" +#include "fsops.h" +#include "comm.h" +#include "symbol_wrapper.h" +#include "qtfs_check.h" + +#define REQ(arg) (arg->data) +#define RSP(arg) (arg->out) +#define USERP(arg) (arg->userp) +DEFINE_MUTEX(fd_bitmap_lock); + +enum { + WHITELIST_MATCH_PREFIX = 0, + WHITELIST_MATCH_EXACT, +}; + +static inline int qtfs_white_list_match(char *path, char *wl, int wl_len, int match_type) +{ + if (strncmp(path, wl, wl_len)) + return 0; + switch (match_type) { + case WHITELIST_MATCH_PREFIX: + if (wl[wl_len - 1] != '/' && path[wl_len] != '\0' && path[wl_len] != '/') + return 0; + break; + case WHITELIST_MATCH_EXACT: + if (path[wl_len] != '\0') + return 0; + break; + default: + return 0; + } + // match success + return 1; +} + +static bool _in_white_list(char *path, int type, int match_type) +{ + int i, in_wl = -1; + char *str; + struct qtfs_wl_cap *cap; + + str = strstr(path, "/.."); + if (str != NULL && (str[3] == '\0' || str[3] =='/')) { + return false; + } + + read_lock(&g_qtfs_wl.rwlock); + cap = &g_qtfs_wl.cap[type]; + for (i = 0; i < cap->nums; i++) { + if (qtfs_white_list_match(path, cap->item[i], strlen(cap->item[i]), match_type)) { + in_wl = i; + break; + } + } + read_unlock(&g_qtfs_wl.rwlock); + return in_wl != -1; +} + +static bool in_white_list(char *path, int type) +{ + return _in_white_list(path, type, WHITELIST_MATCH_PREFIX); +} + +static bool in_white_list_exact(char *path, int type) +{ + return _in_white_list(path, type, WHITELIST_MATCH_EXACT); +} + +static inline void qtfs_inode_info_fill(struct inode_info *ii, struct inode *inode) +{ + ii->mode = inode->i_mode; + ii->i_opflags = inode->i_opflags; + ii->i_uid = inode->i_uid; + ii->i_gid = inode->i_gid; + ii->i_flags = inode->i_flags; + ii->i_ino = inode->i_ino; + ii->i_rdev = inode->i_rdev; + ii->i_size = inode->i_size; + ii->atime = inode->i_atime; + ii->mtime = inode->i_mtime; + ii->ctime = inode->i_ctime; + ii->i_bytes = inode->i_bytes; + ii->i_blkbits = inode->i_blkbits; + ii->i_write_hint = inode->i_write_hint; + ii->i_blocks = inode->i_blocks; + ii->i_state = inode->i_state; + ii->dirtied_when = inode->dirtied_when; + ii->dirtied_time_when = inode->dirtied_time_when; + ii->i_generation = inode->i_generation; + return; +} + +#define QTFS_IOCTL_HANDLE_WITH_BREAK(rspsize)\ + {\ + ret = copy_from_user(rsp->buf, userp->userp, rspsize);\ + if (ret) {\ + qtfs_err("cmd:%d copy_from_user failed with:%d\n", req->d.cmd, ret);\ + rsp->errno = -EFAULT;\ + goto err;\ + }\ + rsp->size = rspsize;\ + break;\ + } +static int handle_ioctl(struct qtserver_arg *arg) +{ + unsigned long ioctl_arg; + int ret; + int iret; + struct qtreq_ioctl *req = (struct qtreq_ioctl *)REQ(arg); + struct qtrsp_ioctl *rsp = (struct qtrsp_ioctl *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + mutex_lock(&fd_bitmap_lock); + if (req->d.fd < 0 || qtfs_fd_bitmap.bitmap == NULL || req->d.fd > qtfs_fd_bitmap.nbits || !test_bit(req->d.fd, qtfs_fd_bitmap.bitmap)) { + mutex_unlock(&fd_bitmap_lock); + qtfs_err("ioctl invalid fd:%d", req->d.fd); + rsp->ret = QTFS_ERR; + rsp->size = 0; + rsp->errno = -EINVAL; + return sizeof(struct qtrsp_ioctl) - sizeof(rsp->buf); + } + mutex_unlock(&fd_bitmap_lock); + + if (req->d.argtype) { + ioctl_arg = req->d.arg; + } else { + if (req->d.size) { + if (req->d.size <= 0 || req->d.size > sizeof(req->path) || req->d.size >= userp->size) { + rsp->errno = -EINVAL; + goto err; + } + ret = copy_to_user(userp->userp, req->path, req->d.size); + if (ret) { + qtfs_err("cmd:%d copy_to_user failed with:%d", req->d.cmd, ret); + rsp->errno = -EFAULT; + goto err; + } + } + ioctl_arg = (unsigned long)userp->userp; + } + iret = qtfs_syscall_ioctl(req->d.fd, req->d.cmd, ioctl_arg); + if (iret) { + qtfs_err("ioctl fd:%d cmd:%d failed with %d", req->d.fd, req->d.cmd, iret); + rsp->errno = iret; + goto err; + } + qtfs_info("ioctl fd:%d cmd:%d argtype:%d arg:%lx size:%u successed", req->d.fd, req->d.cmd, req->d.argtype, req->d.arg, req->d.size); + switch (req->d.cmd) { + case TUNSETPERSIST: + case TUNSETIFF: + case TCSETS: + case FS_IOC_FSSETXATTR: + case SIOCADDMULTI: + case SIOCBRADDBR: + case SIOCBRDELBR: + case SIOCBRADDIF: + case SIOCBRDELIF: + case SIOCDELMULTI: + case SIOCDEVPRIVATE: + case SIOCETHTOOL: + case SIOCSIFFLAGS: + case SIOCSIFHWADDR: + case SIOCSIFMTU: + case SIOCSIFNAME: + rsp->size = 0; + break; + case FS_IOC_FSGETXATTR: + QTFS_IOCTL_HANDLE_WITH_BREAK(sizeof(struct fsxattr)); + case TCGETS: + QTFS_IOCTL_HANDLE_WITH_BREAK(sizeof(struct ktermios)); + case SIOCGIFHWADDR: + case SIOCGIFADDR: + case SIOCGIFFLAGS: + case SIOCGIFINDEX: + case SIOCGIFMTU: + case TUNGETIFF: + QTFS_IOCTL_HANDLE_WITH_BREAK(sizeof(struct ifreq)); + case SIOCGIFVLAN: + QTFS_IOCTL_HANDLE_WITH_BREAK(sizeof(struct vlan_ioctl_args)); + default: + rsp->errno = -EOPNOTSUPP; + goto err; + } + rsp->ret = QTFS_OK; + rsp->errno = iret; + return sizeof(struct qtrsp_ioctl) - sizeof(rsp->buf) + rsp->size; +err: + rsp->ret = QTFS_ERR; + rsp->size = 0; + return sizeof(struct qtrsp_ioctl) - sizeof(rsp->buf); +} + +static int handle_statfs(struct qtserver_arg *arg) +{ + int ret; + struct qtreq_statfs *req = (struct qtreq_statfs *)REQ(arg); + struct qtrsp_statfs *rsp = (struct qtrsp_statfs *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + + if (strlen(req->path) + 1 > userp->size) { + qtfs_err("invalid msg"); + rsp->errno = -EINVAL; + goto err_end; + } + ret = copy_to_user(userp->userp, req->path, strlen(req->path) + 1); + if (ret) { + rsp->errno = -EFAULT; + goto err_end; + } + + ret = qtfs_syscall_statfs((char *)userp->userp, userp->userp2); + if (ret) { + qtfs_err("qtfs server handle statfs path:%s failed with ret:%d.\n", req->path, ret); + rsp->errno = ret; + goto err_end; + } else { + qtfs_info("qtfs server handle statfs path:%s success.\n", req->path); + rsp->ret = QTFS_OK; + } + if (copy_from_user(&rsp->kstat, userp->userp2, sizeof(struct kstatfs))) { + qtfs_err("copy statfs to kstatfs failed"); + rsp->errno = -EFAULT; + goto err_end; + } + return sizeof(struct qtrsp_statfs); +err_end: + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_statfs); +} + +static int handle_mount(struct qtserver_arg *arg) +{ + struct path path; + int ret; + struct qtreq_mount *req = (struct qtreq_mount *)REQ(arg); + struct qtrsp_mount *rsp = (struct qtrsp_mount *)RSP(arg); + if (!in_white_list(req->path, QTFS_WHITELIST_MOUNT)) { + rsp->ret = QTFS_ERR; + rsp->errno = -EPERM; + return sizeof(rsp); + } + + ret = kern_path(req->path, LOOKUP_DIRECTORY, &path); + if (ret) { + qtfs_err("handle mount path:%s kern_path failed, ret: %d.\n", req->path, ret); + rsp->ret = QTFS_ERR; + rsp->errno = -EINVAL; + } else { + rsp->ret = QTFS_OK; + qtfs_info("handle mount path:%s success.\n", req->path); + path_put(&path); + } + return sizeof(rsp); +} + +int handle_open(struct qtserver_arg *arg) +{ + int fd; + int ret; + struct qtreq_open *req = (struct qtreq_open *)REQ(arg); + struct qtrsp_open *rsp = (struct qtrsp_open *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + if (!in_white_list(req->path, QTFS_WHITELIST_OPEN) || qtfs_fd_bitmap.bitmap == NULL) { + qtfs_err("handle open path:%s not permited", req->path); + rsp->ret = QTFS_ERR; + rsp->fd = -EACCES; + return sizeof(struct qtrsp_open); + } + if (strlen(req->path) + 1 > userp->size) { + qtfs_err("path len invalid."); + rsp->ret = QTFS_ERR; + rsp->fd = -EFAULT; + return sizeof(struct qtrsp_open); + } + + ret = copy_to_user(userp->userp, req->path, strlen(req->path)+1); + if (ret) { + qtfs_err("handle open copy to user failed, ret:%d path:%s", ret, req->path); + rsp->ret = QTFS_ERR; + rsp->fd = -EFAULT; + return sizeof(struct qtrsp_open); + } + fd = qtfs_syscall_openat(AT_FDCWD, (char *)userp->userp, req->flags, req->mode); + if (fd == -EEXIST) { + qtfs_err("handle open file <<%s>> flags:%llx mode:%o, opened:failed %d, do again\n", req->path, req->flags, req->mode, fd); + req->flags &= ~(O_CREAT | O_EXCL); + fd = qtfs_syscall_openat(AT_FDCWD, (char *)userp->userp, req->flags, req->mode); + } + if (fd < 0 || fd > qtfs_fd_bitmap.nbits) { + if (fd != -ENOENT) { + qtfs_err("handle open file <<%s>>flags:%llx mode:%o, opened:failed %d\n", req->path, req->flags, req->mode, fd); + } else { + qtfs_info("handle open file <<%s>>flags:%llx mode:%o, opened:failed - file not exist\n", req->path, req->flags, req->mode); + } + rsp->ret = QTFS_ERR; + rsp->fd = (fd < 0) ? fd : -EINVAL; + return sizeof(struct qtrsp_open); + } + mutex_lock(&fd_bitmap_lock); + __set_bit(fd, qtfs_fd_bitmap.bitmap); + mutex_unlock(&fd_bitmap_lock); + rsp->ret = QTFS_OK; + rsp->fd = fd; + return sizeof(struct qtrsp_open); +} + +int handle_close(struct qtserver_arg *arg) +{ + struct qtreq_close *req = (struct qtreq_close *)REQ(arg); + struct qtrsp_close *rsp = (struct qtrsp_close *)RSP(arg); + + // fd >= 3 is valid + if (req->fd <= 2) { + qtfs_err("handle close an invalid fd:%d.", req->fd); + WARN_ON(1); + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_close); + } +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0)) + rsp->ret = qtfs_kern_syms.__close_fd(current->files, req->fd); +#else + rsp->ret = close_fd(req->fd); +#endif + mutex_lock(&fd_bitmap_lock); + if (req->fd > qtfs_fd_bitmap.nbits || qtfs_fd_bitmap.bitmap == NULL || !test_bit(req->fd, qtfs_fd_bitmap.bitmap)) { + qtfs_err("close fd:%d bitmap is notset", req->fd); + } else { + __clear_bit(req->fd, qtfs_fd_bitmap.bitmap); + } + mutex_unlock(&fd_bitmap_lock); + qtfs_info("handle close file, fd:%d ret:%d", req->fd, rsp->ret); + return sizeof(struct qtrsp_close); +} + +static int handle_readiter(struct qtserver_arg *arg) +{ + struct file *file = NULL; + char *pathbuf, *fullname; + int idx = 0; + int ret = 0; + int block_size; + size_t maxlen; + int len; + off_t seek; + struct qtreq_readiter *req = (struct qtreq_readiter *)REQ(arg); + struct qtrsp_readiter *rsp = (struct qtrsp_readiter *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + + mutex_lock(&fd_bitmap_lock); + if (req->fd < 0 || qtfs_fd_bitmap.bitmap == NULL || req->fd > qtfs_fd_bitmap.nbits || !test_bit(req->fd, qtfs_fd_bitmap.bitmap)) { + mutex_unlock(&fd_bitmap_lock); + qtfs_err("unset bitmap fd:%d is error request, fd limit:%u", req->fd, qtfs_fd_bitmap.nbits); + rsp->d.errno = -EINVAL; + goto early_end; + } + mutex_unlock(&fd_bitmap_lock); + file = fget(req->fd); + if (IS_ERR_OR_NULL(file)) { + qtfs_err("handle readiter error, open failed.\n"); + rsp->d.errno = -ENOENT; + goto early_end; + } + if (file->f_flags & O_DIRECT) { + if (file->f_inode->i_sb->s_bdev != NULL && file->f_inode->i_sb->s_bdev->bd_disk != NULL + && file->f_inode->i_sb->s_bdev->bd_disk->queue != NULL) { + block_size = bdev_logical_block_size(file->f_inode->i_sb->s_bdev); + } else { + rsp->d.ret = QTFS_ERR; + rsp->d.errno = -EINVAL; + rsp->d.len = 0; + goto end; + } + if (req->len % block_size != 0) { + rsp->d.ret = QTFS_ERR; + rsp->d.errno = -EINVAL; + rsp->d.len = 0; + goto end; + } + maxlen = (req->len >= sizeof(rsp->readbuf)) ? (block_size * (sizeof(rsp->readbuf) /block_size)) : req->len; + } else { + maxlen = (req->len >= sizeof(rsp->readbuf)) ? (sizeof(rsp->readbuf) - 1) : req->len; + } + + pathbuf = __getname(); + if (pathbuf == NULL) { + qtfs_err("readiter whitelist judge error: failed to get pathbuf.\n"); + rsp->d.ret = QTFS_ERR; + rsp->d.errno = -ENOENT; + goto end; + } + fullname = file_path(file, pathbuf, PATH_MAX); + if (IS_ERR_OR_NULL(fullname) || !in_white_list(fullname, QTFS_WHITELIST_READ)) { + qtfs_err("read iter path not in whitelist.\n"); + __putname(pathbuf); + rsp->d.ret = QTFS_ERR; + rsp->d.errno = -ENOENT; + goto end; + } + qtfs_info("handle readiter file:<%s> len:%lu pos:%lld", fullname, req->len, req->pos); + __putname(pathbuf); + + if (file->f_mode & FMODE_LSEEK) { + seek = qtfs_syscall_lseek(req->fd, req->pos, SEEK_SET); + if (seek < 0) { + qtfs_err("handle read set lseek pos:%lld failed, fd:%d ret:%ld", req->pos, req->fd, seek); + rsp->d.ret = QTFS_ERR; + rsp->d.errno = seek; + fput(file); + return sizeof(struct qtrsp_readiter) - sizeof(rsp->readbuf); + } + } + fput(file); + + rsp->d.ret = QTFS_OK; + while (maxlen > 0) { + len = (maxlen > userp->size) ? userp->size : maxlen; + ret = qtfs_syscall_read(req->fd, userp->userp, len); + if (ret < 0) { + qtfs_err("read fd:%d failed:%d", req->fd, ret); + rsp->d.ret = QTFS_ERR; + break; + } + if (ret == 0) { + rsp->d.end = 1; + break; + } + maxlen -= len; + if (copy_from_user(&rsp->readbuf[idx], userp->userp, ret)) { + qtfs_err("copy from user failed fd:%d len:%d", req->fd, ret); + rsp->d.end = 1; + rsp->d.ret = QTFS_ERR; + break; + } + idx += ret; + rsp->d.len += ret; + if (ret < len) { + rsp->d.end = 1; + break; + } + } + + qtfs_info("read fd:%d len:%ld %s errno:%d", req->fd, rsp->d.len, + (rsp->d.ret == QTFS_OK) ? "Successed" : "Failed", rsp->d.errno); + + return sizeof(struct qtrsp_readiter) - sizeof(rsp->readbuf) + ((rsp->d.len < 0) ? 0 : rsp->d.len); +end: + fput(file); + return sizeof(struct qtrsp_readiter) - sizeof(rsp->readbuf) + ((rsp->d.len < 0) ? 0 : rsp->d.len); +early_end: + rsp->d.ret = QTFS_ERR; + rsp->d.len = 0; + return sizeof(struct qtrsp_readiter) - sizeof(rsp->readbuf); +} + +static int handle_write(struct qtserver_arg *arg) +{ + struct file *file = NULL; + char *pathbuf, *fullname; + int block_size; + struct qtreq_write *req = (struct qtreq_write *)REQ(arg); + struct qtrsp_write *rsp = (struct qtrsp_write *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + int idx = 0, leftlen = 0, ret = 0, len = 0; + off_t seek; + + mutex_lock(&fd_bitmap_lock); + if (req->d.fd < 0 || qtfs_fd_bitmap.bitmap == NULL || req->d.fd > qtfs_fd_bitmap.nbits || !test_bit(req->d.fd, qtfs_fd_bitmap.bitmap)) { + mutex_unlock(&fd_bitmap_lock); + qtfs_err("unset bitmap fd:%d is error request, fd limit:%u", req->d.fd, qtfs_fd_bitmap.nbits); + goto early_end; + } + mutex_unlock(&fd_bitmap_lock); + file = fget(req->d.fd); + if (IS_ERR_OR_NULL(file)) { + qtfs_err("qtfs handle write error, open failed.\n"); + goto early_end; + } + if (file->f_flags & O_DIRECT) { + if (file->f_inode->i_sb->s_bdev != NULL && file->f_inode->i_sb->s_bdev->bd_disk != NULL + && file->f_inode->i_sb->s_bdev->bd_disk->queue != NULL) { + block_size = bdev_logical_block_size(file->f_inode->i_sb->s_bdev); + } else { + rsp->ret = QTFS_ERR; + rsp->len = -EINVAL; + goto end; + } + if (req->d.total_len % block_size != 0) { + rsp->ret = QTFS_ERR; + rsp->len = -EINVAL; + goto end; + } + leftlen = block_size * (req->d.buflen / block_size); + } else { + leftlen = req->d.buflen; + } + if (leftlen < 0 || leftlen > sizeof(req->path_buf)) { + qtfs_err("invalid buflen :%d", leftlen); + rsp->ret = QTFS_ERR; + rsp->len = -EINVAL; + goto end; + } + pathbuf = __getname(); + if (pathbuf == NULL) { + qtfs_err("write whitelist judge error: failed to get pathbuf.\n"); + rsp->ret = QTFS_ERR; + rsp->len = 0; + goto end; + } + fullname = file_path(file, pathbuf, PATH_MAX); + if (IS_ERR_OR_NULL(fullname) || !in_white_list(fullname, QTFS_WHITELIST_WRITE)) { + __putname(pathbuf); + rsp->ret = QTFS_ERR; + rsp->len = 0; + goto end; + } + qtfs_info("handle write fd:%d file:<%s>, write len:%d before pos:%lld mode:%o flags:%x", req->d.fd, fullname, + leftlen, req->d.pos, file->f_mode, file->f_flags); + __putname(pathbuf); + + if (file->f_mode & FMODE_LSEEK) { + seek = qtfs_syscall_lseek(req->d.fd, req->d.pos, SEEK_SET); + if (seek < 0) { + qtfs_err("handle write set lseek pos:%lld failed, fd:%d ret:%ld", req->d.pos, req->d.fd, seek); + rsp->ret = QTFS_ERR; + rsp->len = seek; + fput(file); + return sizeof(struct qtrsp_write); + } + } + fput(file); + + rsp->ret = QTFS_OK; + while (leftlen > 0) { + len = (leftlen > userp->size) ? userp->size : leftlen; + leftlen -= len; + if (copy_to_user(userp->userp, &req->path_buf[idx], len)) { + qtfs_err("copy to user failed len:%d idx:%d", len, idx); + rsp->ret = QTFS_ERR; + break; + } + idx += len; + ret = qtfs_syscall_write(req->d.fd, userp->userp, len); + if (ret <= 0) { + qtfs_err("write failed ret:%d", ret); + rsp->ret = QTFS_ERR; + break; + } + rsp->len += ret; + } + + qtfs_info("write fd:%d len:%ld %s", req->d.fd, rsp->len, (rsp->ret == QTFS_OK) ? "Successed" : "Failed"); + return sizeof(struct qtrsp_write); +end: + fput(file); + return sizeof(struct qtrsp_write); +early_end: + rsp->ret = QTFS_ERR; + rsp->len = 0; + return sizeof(struct qtrsp_write); +} + +static int handle_lookup(struct qtserver_arg *arg) +{ + struct path path; + struct inode *inode; + struct qtreq_lookup *req = (struct qtreq_lookup *)REQ(arg); + struct qtrsp_lookup *rsp = (struct qtrsp_lookup *)RSP(arg); + int ret; + ret = kern_path(req->fullname, 0, &path); + if (ret) { + qtfs_info("qtfs handle lookup(%s) kern_path failed, ret%d.\n", req->fullname, ret); + rsp->errno = (ret == -ENOENT ? 0 : ret); + rsp->ret = QTFS_ERR; + } else { + inode = path.dentry->d_inode; + rsp->ret = QTFS_OK; + qtfs_inode_info_fill(&rsp->inode_info, inode); + qtfs_debug("handle lookup name:%s, mode:%o ino:%lu", req->fullname, rsp->inode_info.mode, rsp->inode_info.i_ino); + path_put(&path); + } + return sizeof(struct qtrsp_lookup); +} +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)) +static bool qtfs_filldir(struct dir_context *ctx, const char *name, int namelen, + loff_t offset, u64 ino, unsigned int d_type) +#else +static int qtfs_filldir(struct dir_context *ctx, const char *name, int namelen, + loff_t offset, u64 ino, unsigned int d_type) +#endif +{ + struct qtfs_dirent64 *dirent, *prev; + struct qtfs_getdents *buf = container_of(ctx, struct qtfs_getdents, ctx); + int reclen = ALIGN(offsetof(struct qtfs_dirent64, d_name) + namelen + 1, sizeof(u64)); + int prev_reclen; + + if (reclen > buf->count || !buf->dir || !name) +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)) + return false; +#else + return -EINVAL; +#endif + + prev_reclen = buf->prev_reclen; + dirent = buf->dir; + prev = (void *)dirent - prev_reclen; + prev->d_off = offset; + dirent->d_ino = ino; + dirent->d_reclen = reclen; + dirent->d_type = d_type; + memcpy(dirent->d_name, name, namelen); + + buf->prev_reclen = reclen; + buf->dir = (void *)dirent + reclen; + buf->count -= reclen; + buf->vldcnt++; +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)) + return true; +#else + return 0; +#endif +} + +static int handle_readdir(struct qtserver_arg *arg) +{ + struct file *file = NULL; + struct qtreq_readdir *req = (struct qtreq_readdir *)REQ(arg); + struct qtrsp_readdir *rsp = (struct qtrsp_readdir *)RSP(arg); + int ret; + struct qtfs_getdents buf = { + .ctx.actor = qtfs_filldir, + .ctx.pos = req->pos, + .prev_reclen = 0, + .count = req->count, + .dir = (struct qtfs_dirent64 *)rsp->dirent, + .vldcnt = 0, + }; + + if (!in_white_list(req->path, QTFS_WHITELIST_READDIR)) { + rsp->d.ret = QTFS_ERR; + rsp->d.vldcnt = 0; + return sizeof(struct qtrsp_readdir) - sizeof(rsp->dirent); + } + file = filp_open(req->path, O_RDONLY|O_NONBLOCK|O_DIRECTORY, 0); + if (IS_ERR_OR_NULL(file)) { + qtfs_err("handle readdir error, filp:<%s> open failed.\n", req->path); + rsp->d.ret = QTFS_ERR; + rsp->d.vldcnt = 0; + return sizeof(struct qtrsp_readdir) - sizeof(rsp->dirent); + } + file->f_pos = req->pos; + ret = iterate_dir(file, &buf.ctx); + rsp->d.pos = file->f_pos; + rsp->d.ret = QTFS_OK; + rsp->d.vldcnt = buf.vldcnt; + rsp->d.over = (req->pos == rsp->d.pos) ? 1 : 0; + qtfs_info("handle readdir ret:%d, pos:%lld path:%s, valid count:%d, leftcount:%d validbyte:%lu\n", + ret, req->pos, req->path, buf.vldcnt, buf.count, sizeof(rsp->dirent) - buf.count); + filp_close(file, NULL); + + return sizeof(struct qtrsp_readdir) - buf.count; +} + +static int handle_mkdir(struct qtserver_arg *arg) +{ + struct qtreq_mkdir *req = (struct qtreq_mkdir *)REQ(arg); + struct qtrsp_mkdir *rsp = (struct qtrsp_mkdir *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + struct inode *inode; + struct path path; + int ret; + int len; + + if (!in_white_list(req->path, QTFS_WHITELIST_MKDIR)) { + rsp->errno = -EFAULT; + goto err; + } + len = strlen(req->path); + if (len < 0 || len + 1 > userp->size || len >= sizeof(req->path)) { + qtfs_err("path len invalid:%d.", len); + rsp->errno = -EFAULT; + goto err; + + } + if (copy_to_user(userp->userp, req->path, len + 1)) { + qtfs_err("handle mkdir copy to userp failed.\n"); + rsp->errno = -EFAULT; + goto err; + } + rsp->errno = qtfs_syscall_mkdirat(AT_FDCWD, userp->userp, req->mode); + if (rsp->errno < 0) { + qtfs_err("handle mkdir path:%s failed with ret:%d.", req->path, rsp->errno); + goto err; + } + ret = kern_path(req->path, 0, &path); + if (ret) { + qtfs_err("handle mkdir failed in kern path, ret:%d.\n", ret); + rsp->errno = -EFAULT; + goto err; + } else { + inode = d_inode(path.dentry); + qtfs_inode_info_fill(&rsp->inode_info, inode); + path_put(&path); + } + rsp->ret = QTFS_OK; + qtfs_info("handle mkdir path:%s success.", req->path); + return sizeof(struct qtrsp_mkdir); + +err: + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_mkdir); +} + +static int handle_rmdir(struct qtserver_arg *arg) +{ + struct qtreq_rmdir *req = (struct qtreq_rmdir *)REQ(arg); + struct qtrsp_rmdir *rsp = (struct qtrsp_rmdir *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + int len; + + if (!in_white_list(req->path, QTFS_WHITELIST_RMDIR)) { + rsp->errno = -EFAULT; + goto err; + } + len = strlen(req->path); + if (len < 0 || len + 1 > userp->size || len >= sizeof(req->path)) { + qtfs_err("len invalid:%d", len); + rsp->errno = -EFAULT; + goto err; + } + if (copy_to_user(userp->userp, req->path, len + 1)) { + qtfs_err("handle rmdir copy to userp failed.\n"); + rsp->errno = -EFAULT; + goto err; + } + rsp->errno = qtfs_syscall_rmdir(userp->userp); + if (rsp->errno < 0) { + qtfs_err("handle rmdir error:%d.", rsp->errno); + goto err; + } + qtfs_info("handle rmdir path:%s success.", req->path); + rsp->ret = QTFS_OK; + return sizeof(struct qtrsp_rmdir); + +err: + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_rmdir); +} + +static int handle_getattr(struct qtserver_arg *arg) +{ + struct qtreq_getattr *req = (struct qtreq_getattr *)REQ(arg); + struct qtrsp_getattr *rsp = (struct qtrsp_getattr *)RSP(arg); + struct path path; + int ret; + + qtfs_debug("handle getattr path:%s\n", req->path); + ret = kern_path(req->path, 0, &path); + if (ret) { + rsp->errno = ret; + qtfs_err("handle getattr path:%s failed, ret:%d %s\n", req->path, ret, (ret != -ENOENT) ? "." : "file not exist"); + goto failed; + } + + ret = vfs_getattr(&path, &rsp->stat, req->request_mask, req->query_flags); + if (ret) { + qtfs_err("vfs getattr path:%s ret:%d\n", req->path, ret); + rsp->errno = ret; + path_put(&path); + goto failed; + } + rsp->ret = QTFS_OK; + path_put(&path); + qtfs_debug("handle getattr:<%s> blksize:%u size:%lld mode:%o ino:%llu req_mask:%x req_flags:%u.\n", req->path, rsp->stat.blksize, + rsp->stat.size, rsp->stat.mode, rsp->stat.ino, req->request_mask, req->query_flags); + return sizeof(struct qtrsp_getattr); + +failed: + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_getattr); +} + +static int handle_setattr(struct qtserver_arg *arg) +{ + struct qtreq_setattr *req = (struct qtreq_setattr *)REQ(arg); + struct qtrsp_setattr *rsp = (struct qtrsp_setattr *)RSP(arg); + struct inode *inode = NULL; + struct path path; + int ret; + + if (!in_white_list(req->path, QTFS_WHITELIST_SETATTR)) { + rsp->ret = QTFS_ERR; + rsp->errno = -ENOENT; + return sizeof(struct qtrsp_setattr); + } + + ret = kern_path(req->path, 0, &path); + if (ret) { + qtfs_err("handle setattr path:%s failed in kern_path with %d\n", req->path, ret); + rsp->ret = QTFS_ERR; + rsp->errno = -ENOENT; + return sizeof(struct qtrsp_setattr); + } + inode = path.dentry->d_inode; + if (req->attr.ia_valid & (ATTR_KILL_SUID | ATTR_KILL_SGID)) { + req->attr.ia_valid &= ~(ATTR_KILL_SUID | ATTR_KILL_SGID | + ATTR_MODE); + req->attr.ia_mode = inode->i_mode; + if (inode->i_mode & S_ISUID) { + req->attr.ia_valid |= ATTR_MODE; + req->attr.ia_mode &= ~S_ISUID; + } + if ((inode->i_mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP)) { + req->attr.ia_valid |= ATTR_MODE; + req->attr.ia_mode &= ~S_ISGID; + } + } + if (!req->attr.ia_valid) { + rsp->ret = QTFS_OK; + path_put(&path); + return sizeof(struct qtrsp_setattr); + } + + inode_lock(inode); +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) + rsp->errno = notify_change(&nop_mnt_idmap, path.dentry, &req->attr, NULL); +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) + rsp->errno = notify_change(&init_user_ns, path.dentry, &req->attr, NULL); +#else + rsp->errno = notify_change(path.dentry, &req->attr, NULL); +#endif + if (rsp->errno < 0) { + rsp->ret = QTFS_ERR; + qtfs_err("handle setattr, path:<%s> failed with %d.\n", req->path, ret); + goto end; + } + + qtfs_info("handle setattr iattr success iavalid:%u mode:%o size:%lld\n", + req->attr.ia_valid, req->attr.ia_mode, req->attr.ia_size); + rsp->ret = QTFS_OK; + +end: + inode_unlock(inode); + path_put(&path); + return sizeof(struct qtrsp_setattr); +} + +int handle_icreate(struct qtserver_arg *arg) +{ + struct file *file = NULL; + struct inode *inode; + struct qtreq_icreate *req = (struct qtreq_icreate *)REQ(arg); + struct qtrsp_icreate *rsp = (struct qtrsp_icreate *)RSP(arg); + + if (!in_white_list(req->path, QTFS_WHITELIST_CREATE)) { + rsp->ret = QTFS_ERR; + rsp->errno = -ENOENT; + return sizeof(struct qtrsp_icreate); + } + + file = filp_open(req->path, O_CREAT, req->mode); + if (IS_ERR_OR_NULL(file)) { + qtfs_err("handle icreate filp:<%s> failed in open.\n", req->path); + rsp->ret = QTFS_ERR; + rsp->errno = QTFS_PTR_ERR(file); + return sizeof(struct qtrsp_icreate); + } + inode = file->f_inode; + qtfs_inode_info_fill(&rsp->inode_info, inode); + filp_close(file, NULL); + rsp->ret = QTFS_OK; + qtfs_info("handle icreate path:%s success, inode mode:%ho\n", req->path, + rsp->inode_info.mode); + return sizeof(struct qtrsp_icreate); +} + +static int handle_mknod(struct qtserver_arg *arg) +{ + struct qtreq_mknod *req = (struct qtreq_mknod *)REQ(arg); + struct qtrsp_mknod *rsp = (struct qtrsp_mknod *)RSP(arg); + struct dentry *dent = NULL; + struct path path; + int error; + unsigned int flags = LOOKUP_DIRECTORY; + + if (!in_white_list(req->path, QTFS_WHITELIST_CREATE)) { + rsp->ret = QTFS_ERR; + rsp->errno = -ENOENT; + return sizeof(struct qtrsp_mknod); + } + +retry: + dent = kern_path_create(AT_FDCWD, req->path, &path, flags); + if (IS_ERR_OR_NULL(dent)) { + rsp->ret = QTFS_ERR; + rsp->errno = QTFS_PTR_ERR(dent); + qtfs_info("handle mknod path:<%s>, mode:%o in kern_path_create with ret:%ld\n", req->path, req->mode, QTFS_PTR_ERR(dent)); + return sizeof(struct qtrsp_mknod); + } + + if (!IS_POSIXACL(path.dentry->d_inode)) + req->mode &= ~current_umask(); + error = security_path_mknod(&path, dent, req->mode, req->dev); + if (!error) +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) + error = vfs_mknod(&nop_mnt_idmap, path.dentry->d_inode, dent, req->mode, req->dev); +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) + error = vfs_mknod(&init_user_ns, path.dentry->d_inode, dent, req->mode, req->dev); +#else + error = vfs_mknod(path.dentry->d_inode, dent, req->mode, req->dev); +#endif + done_path_create(&path, dent); + if (error == -ESTALE && !(flags & LOOKUP_REVAL)) { + flags |= LOOKUP_REVAL; + qtfs_debug("retry mknod.\n"); + rsp->errno = error; + goto retry; + } + qtfs_inode_info_fill(&rsp->inode_info, dent->d_inode); + rsp->ret = QTFS_OK; + qtfs_info("handle mknod path:<%s>, mode:%o success\n", req->path, req->mode); + rsp->errno = 0; + return sizeof(struct qtrsp_mknod); +} + +int handle_unlink(struct qtserver_arg *arg) +{ + struct qtreq_unlink *req = (struct qtreq_unlink *)REQ(arg); + struct qtrsp_unlink *rsp = (struct qtrsp_unlink *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + int len; + + if (!in_white_list(req->path, QTFS_WHITELIST_UNLINK)) { + rsp->errno = -ENOENT; + return sizeof(struct qtrsp_unlink); + } + len = strlen(req->path); + if (len < 0 || len + 1 > userp->size || len >= sizeof(req->path)) { + qtfs_err("len invalid:%d", len); + rsp->errno = -EFAULT; + return sizeof(struct qtrsp_unlink); + } + if (copy_to_user(userp->userp, req->path, len + 1)) { + qtfs_err("handle unlink copy to userp failed."); + rsp->errno = -EFAULT; + return sizeof(struct qtrsp_unlink); + } + + rsp->errno = qtfs_syscall_unlink(userp->userp); + if (rsp->errno < 0) { + qtfs_err("handle unlink failed, errno:%d\n", rsp->errno); + } else { + qtfs_info("handle unlink path:%s success\n", req->path); + } + return sizeof(struct qtrsp_unlink); +} + +int handle_link(struct qtserver_arg *arg) +{ + char *oldname, *newname; + struct qtreq_link *req = (struct qtreq_link *)REQ(arg); + struct qtrsp_link *rsp = (struct qtrsp_link *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + + oldname = req->path; + newname = req->path + req->d.oldlen; + if (strlen(oldname) >= sizeof(req->path) || req->d.oldlen >= sizeof(req->path) || + strlen(oldname) + strlen(newname) >= sizeof(req->path) || + strlen(oldname) < 0 || strlen(newname) < 0) { + qtfs_err("invalid path oldname or newname during handle_link"); + rsp->errno = -EFAULT; + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_link); + } + if (strlen(oldname) + 1 > userp->size || strlen(newname) + 1 > userp->size || + copy_to_user(userp->userp, oldname, strlen(oldname) + 1) || + copy_to_user(userp->userp2, newname, strlen(newname) + 1)) { + qtfs_err("handle link failed in copy to userp.\n"); + rsp->errno = -EFAULT; + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_link); + } + + rsp->errno = qtfs_syscall_linkat(AT_FDCWD, userp->userp, AT_FDCWD, userp->userp2, 0); + qtfs_info("handle link new:%s old:%s return %d\n", newname, oldname, rsp->errno); + rsp->ret = rsp->errno == 0 ? QTFS_OK : QTFS_ERR; + return sizeof(struct qtrsp_link); +} + +int handle_symlink(struct qtserver_arg *arg) +{ + char *oldname, *newname; + struct qtreq_symlink *req = (struct qtreq_symlink *)REQ(arg); + struct qtrsp_symlink *rsp = (struct qtrsp_symlink *)RSP(arg); + int error; + struct dentry *dentry; + struct path path; + unsigned int lookup_flags = 0; + + if (req->d.newlen >= sizeof(req->path) || req->d.newlen + req->d.oldlen > sizeof(req->path)) { + qtfs_err("newlen:%lu oldlen:%lu is too big", req->d.newlen, req->d.oldlen); + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_symlink); + } + newname = req->path; + oldname = &req->path[req->d.newlen]; +retry: + dentry = kern_path_create(AT_FDCWD, newname, &path, lookup_flags); + error = QTFS_PTR_ERR(dentry); + if (IS_ERR_OR_NULL(dentry)) { + rsp->ret = QTFS_ERR; + qtfs_err("handle_symlink: newname(%s), oldname(%s) in kern_path_create %d\n", newname, oldname, error); + return sizeof(struct qtrsp_symlink); + } +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) + rsp->errno = vfs_symlink(&nop_mnt_idmap, path.dentry->d_inode, dentry, oldname); +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) + rsp->errno = vfs_symlink(&init_user_ns, path.dentry->d_inode, dentry, oldname); +#else + rsp->errno = vfs_symlink(path.dentry->d_inode, dentry, oldname); +#endif + done_path_create(&path, dentry); + if (rsp->errno == -ESTALE && !(lookup_flags & LOOKUP_REVAL)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + rsp->ret = QTFS_OK; + qtfs_info("handle_symlink: newname(%s), oldname(%s) success\n", newname, oldname); + qtfs_inode_info_fill(&rsp->inode_info, dentry->d_inode); + return sizeof(struct qtrsp_symlink); +} + +int handle_getlink(struct qtserver_arg *arg) +{ + struct qtreq_getlink *req = (struct qtreq_getlink *)REQ(arg); + struct qtrsp_getlink *rsp = (struct qtrsp_getlink *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + + if (strlen(req->path) < 0 || strlen(req->path) + 1 > sizeof(req->path) || copy_to_user(userp->userp, req->path, strlen(req->path) + 1)) { + qtfs_err("handle getlink<%s> copy to userp failed.\n", req->path); + rsp->errno = -EFAULT; + goto err_handle; + } + rsp->errno = qtfs_syscall_readlinkat(AT_FDCWD, userp->userp, userp->userp2, userp->size); + if (rsp->errno < 0 || rsp->errno > MAX_PATH_LEN) { + qtfs_err("handle getlink<%s> do readlinkat failed, errno:%d\n", req->path, rsp->errno); + goto err_handle; + } + if (copy_from_user(rsp->path, userp->userp2, rsp->errno)) { + qtfs_err("handle getlink<%s> copy from user failed, len:%d.", req->path, rsp->errno); + rsp->errno = -EFAULT; + goto err_handle; + } + rsp->ret = QTFS_OK; + qtfs_info("handle getlink<%s> ok, len:%d link:%s.", req->path, rsp->errno, rsp->path); + return sizeof(struct qtrsp_getlink) - sizeof(rsp->path) + strlen(rsp->path) + 1; + +err_handle: + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_getlink) - sizeof(rsp->path); +} + +int handle_rename(struct qtserver_arg *arg) +{ + struct qtreq_rename *req = (struct qtreq_rename *)REQ(arg); + struct qtrsp_rename *rsp = (struct qtrsp_rename *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + + if (!in_white_list(req->path, QTFS_WHITELIST_RENAME)) { + rsp->errno = -ENOENT; + goto err_handle; + } + if (strlen(req->path) + 1 > sizeof(req->path) || req->d.oldlen >= sizeof(req->path) || + strlen(req->path) + strlen(&req->path[req->d.oldlen]) >= sizeof(req->path) || + strlen(req->path) + 1 > userp->size || strlen(&req->path[req->d.oldlen]) + 1 > userp->size) { + qtfs_err("invalid req msg"); + rsp->errno = -EFAULT; + goto err_handle; + } + if (copy_to_user(userp->userp, req->path, strlen(req->path) + 1) || + copy_to_user(userp->userp2, &req->path[req->d.oldlen], strlen(&req->path[req->d.oldlen]) + 1)) { + qtfs_err("handle rename copy to userp failed.\n"); + rsp->errno = -EFAULT; + goto err_handle; + } + rsp->errno = qtfs_syscall_renameat2(AT_FDCWD, userp->userp, AT_FDCWD, userp->userp2, 0); + +err_handle: + rsp->ret = (rsp->errno < 0) ? QTFS_ERR : QTFS_OK; + qtfs_info("handle rename oldname:%s newname:%s ret:%d %s", req->path, &req->path[req->d.oldlen], rsp->errno, + (rsp->errno < 0) ? "failed" : "successed"); + return sizeof(struct qtrsp_rename); +} + +int handle_xattrlist(struct qtserver_arg *arg) +{ + struct qtreq_xattrlist *req = (struct qtreq_xattrlist *)REQ(arg); + struct qtrsp_xattrlist *rsp = (struct qtrsp_xattrlist *)RSP(arg); + struct path path; + int ret = 0; + ssize_t size = 0, buffer_size = 0; + int i = 0; + + buffer_size = (req->buffer_size > sizeof(rsp->name)) ? sizeof(rsp->name) : req->buffer_size; + ret = kern_path(req->path, 0, &path); + if (ret) { + qtfs_err("handle xattr list path error.\n"); + rsp->d.size = -ENOENT; + goto err_handle; + } + size = vfs_listxattr(path.dentry, buffer_size == 0 ? NULL : rsp->name, buffer_size); + path_put(&path); + if (size < 0) { + qtfs_err("handle list xattr failed, errno:%ld.\n", size); + rsp->d.size = size; + goto err_handle; + } + if (size == 0) { + rsp->d.size = size; + goto err_handle; + } + rsp->d.ret = QTFS_OK; + rsp->d.size = size; + while (i < size) { + qtfs_info("handle list xattr result:%s\n", &rsp->name[i]); + i += strlen(&rsp->name[i]) + 1; + } + return sizeof(struct qtrsp_xattrlist); + +err_handle: + rsp->d.ret = QTFS_ERR; + return sizeof(struct qtrsp_xattrlist); +} + +int handle_xattrset(struct qtserver_arg *arg) +{ + struct qtreq_xattrset *req = (struct qtreq_xattrset *)REQ(arg); + struct qtrsp_xattrset *rsp = (struct qtrsp_xattrset *)RSP(arg); + struct path path; + int ret = 0; + + if (!in_white_list(req->buf, QTFS_WHITELIST_SETXATTR)) { + rsp->errno = -ENOENT; + goto err_handle; + } + + ret = kern_path(req->buf, 0, &path); + if (ret) { + qtfs_err("handle xattrset path error, file:%s.\n", req->buf); + rsp->errno = -ENOENT; + goto err_handle; + } + if (req->d.pathlen + req->d.namelen + req->d.valuelen > sizeof(req->buf) - 3) { + qtfs_err("invalid len:%lu %lu %lu", req->d.pathlen, req->d.namelen, req->d.valuelen); + rsp->errno = -EFAULT; + path_put(&path); + goto err_handle; + } +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) + rsp->errno = vfs_setxattr(&nop_mnt_idmap, path.dentry, &req->buf[req->d.pathlen], &req->buf[req->d.pathlen + req->d.namelen], req->d.valuelen, req->d.flags); +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) + rsp->errno = vfs_setxattr(&init_user_ns, path.dentry, &req->buf[req->d.pathlen], &req->buf[req->d.pathlen + req->d.namelen], req->d.valuelen, req->d.flags); +#else + rsp->errno = vfs_setxattr(path.dentry, &req->buf[req->d.pathlen], &req->buf[req->d.pathlen + req->d.namelen], req->d.valuelen, req->d.flags); +#endif + qtfs_info("handle xattrset path:%s name:%s value:%s ret:%d size:%lu flags:%d", req->buf, + &req->buf[req->d.pathlen], &req->buf[req->d.pathlen + req->d.namelen], rsp->errno, + req->d.valuelen, req->d.flags); + path_put(&path); + return sizeof(struct qtrsp_xattrset); + +err_handle: + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_xattrset); +} + +int handle_xattrget(struct qtserver_arg *arg) +{ + struct qtreq_xattrget *req = (struct qtreq_xattrget *)REQ(arg); + struct qtrsp_xattrget *rsp = (struct qtrsp_xattrget *)RSP(arg); + struct path path; + int ret = 0; + ssize_t error = 0; + int len = 0; + char *kvalue = NULL; + + ret = kern_path(req->path, 0, &path); + if (ret) { + qtfs_err("handle xattrget path error.\n"); + rsp->d.errno = -ENOENT; + goto err_handle; + } + + if (req->d.size > 0) { + if (req->d.size > XATTR_SIZE_MAX) + req->d.size = XATTR_SIZE_MAX; + + if (req->d.pos > req->d.size) { + rsp->d.errno = -EINVAL; + path_put(&path); + goto err_handle; + } + kvalue = (char *)kvzalloc(req->d.size, GFP_KERNEL); + if (!kvalue) { + qtfs_err("handle xattrget kvzalloc failed, size:%d.\n", req->d.size); + rsp->d.errno = -ENOMEM; + path_put(&path); + goto err_handle; + } + } +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) + error = vfs_getxattr(&nop_mnt_idmap, path.dentry, req->d.prefix_name, kvalue, req->d.size); +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0)) + error = vfs_getxattr(&init_user_ns, path.dentry, req->d.prefix_name, kvalue, req->d.size); +#else + error = vfs_getxattr(path.dentry, req->d.prefix_name, kvalue, req->d.size); +#endif + path_put(&path); + if (error > 0) { + if (req->d.pos >= error) { + rsp->d.size = 0; + rsp->d.pos = req->d.pos; + goto end; + } + qtfs_info("handle getxattr: path:%s prefix name:%s : (%s - 0x%llx), size:%ld, reqpos:%d\n", req->path, req->d.prefix_name, kvalue, (__u64)kvalue, error, req->d.pos); + len = (error - req->d.pos) > sizeof(rsp->buf) ? sizeof(rsp->buf) : (error - req->d.pos); + rsp->d.size = len; + if (req->d.size > 0) { + memcpy(rsp->buf, &kvalue[req->d.pos], len); + } + rsp->d.pos = req->d.pos + len; + } else { + rsp->d.errno = error; + kvfree(kvalue); + goto err_handle; + } +end: + qtfs_info("handle getxattr successed file:%s result:%s", req->path, rsp->buf); + kvfree(kvalue); + rsp->d.ret = QTFS_OK; + return sizeof(struct qtrsp_xattrget) - sizeof(rsp->buf) + len; + +err_handle: + rsp->d.ret = QTFS_ERR; + qtfs_err("handle getxattr failed, file:%s", req->path); + return sizeof(struct qtrsp_xattrget) - sizeof(rsp->buf); +} + +int handle_syscall_mount(struct qtserver_arg *arg) +{ + struct qtreq_sysmount *req = (struct qtreq_sysmount *)REQ(arg); + struct qtrsp_sysmount *rsp = (struct qtrsp_sysmount *)RSP(arg); + char *dev_name, *dir_name, *type; + char *udir_name = NULL; + void *data_page; + char *udev_name = NULL; + char *utype = NULL; + char *udata = NULL; + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + + if (req->d.dev_len + req->d.dir_len + req->d.type_len + req->d.data_len > sizeof(req->buf) - 4) { + qtfs_err("invalid msglen:%lu %lu %lu %lu", req->d.dev_len, req->d.dir_len, req->d.type_len, req->d.data_len); + rsp->errno = -EINVAL; + goto end; + } + dev_name = req->d.dev_len == 0 ? NULL : req->buf; + dir_name = &req->buf[req->d.dev_len]; + type = req->d.type_len == 0 ? NULL : &req->buf[req->d.dev_len + req->d.dir_len]; + if (req->d.data_len != 0) + data_page = &req->buf[req->d.dev_len + req->d.dir_len + req->d.type_len]; + else + data_page = NULL; + + if (strlen(dir_name) >= (userp->size / 2) || + (dev_name != NULL && strlen(dev_name) >= (userp->size / 2)) || + (type != NULL && strlen(type) >= (userp->size / 2)) || + (data_page != NULL && strlen(data_page) >= (userp->size / 2))) { + qtfs_err("mount str len is too big dirname:%lu devname:%lu type:%lu data:%lu", + strlen(dir_name), (dev_name == NULL) ? 0 : strlen(dev_name), + (type == NULL) ? 0 : strlen(type), + (data_page == NULL) ? 0 : strlen(data_page)); + rsp->errno = -EINVAL; + goto end; + } + udir_name = userp->userp; + udev_name = (dev_name == NULL) ? NULL : ((char *)userp->userp + userp->size / 2); + utype = (type == NULL) ? NULL : userp->userp2; + udata = (data_page == NULL) ? NULL : ((char *)userp->userp + userp->size / 2); + + if (copy_to_user(udir_name, dir_name, strlen(dir_name) + 1) || + (dev_name != NULL && copy_to_user(udev_name, dev_name, strlen(dev_name) + 1)) || + (type != NULL && copy_to_user(utype, type, strlen(type) + 1)) || + (data_page != NULL && copy_to_user(udata, data_page, strlen(data_page) + 1))) { + qtfs_err("syscall mount failed to copy to user"); + rsp->errno = -ENOMEM; + goto end; + } + + qtfs_info("handle syscall mount devname:%s dirname:%s type:%s data:%s\n", dev_name, dir_name, type, + (data_page == NULL) ? "nil" : (char *)data_page); + rsp->errno = qtfs_syscall_mount(udev_name, udir_name, utype, req->d.flags, udata); + if (rsp->errno < 0) + qtfs_err("handle syscall mount failed devname:%s dirname:%s type:%s data:%s, errno:%d\n", + dev_name, dir_name, type, (char *)data_page, rsp->errno); + +end: + return sizeof(struct qtrsp_sysmount); +} + +int handle_syscall_umount(struct qtserver_arg *arg) +{ + struct qtreq_sysumount *req = (struct qtreq_sysumount *)REQ(arg); + struct qtrsp_sysumount *rsp = (struct qtrsp_sysumount *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + + qtfs_info("handle umount path:%s\n", req->buf); + if (strlen(req->buf) + 1 > userp->size || strlen(req->buf) >= sizeof(req->buf)) { + qtfs_err("invalid msg"); + rsp->errno = -EINVAL; + return sizeof(struct qtrsp_sysumount); + } + if (copy_to_user(userp->userp, req->buf, strlen(req->buf) + 1)) { + rsp->errno = -ENOMEM; + return sizeof(struct qtrsp_sysumount); + } + rsp->errno = qtfs_syscall_umount(userp->userp, req->flags); + if (rsp->errno) + qtfs_err("umount(%s) failed, errno:%d\n", req->buf, rsp->errno); + //dont need to path_put here. + return sizeof(struct qtrsp_sysumount); +} + +#ifdef KVER_4_19 +static bool qtfs_pipe_empty(struct pipe_inode_info *pipe) +{ + int nrbufs; + + nrbufs = pipe->nrbufs; + return nrbufs <= 0; +} + +static bool qtfs_pipe_full(struct pipe_inode_info *pipe) +{ + int nrbufs; + + nrbufs = pipe->nrbufs; + return nrbufs >= pipe->buffers; +} +#else +static bool qtfs_pipe_empty(struct pipe_inode_info *pipe) +{ + unsigned int head, tail; + + head = READ_ONCE(pipe->head); + tail = READ_ONCE(pipe->tail); + + return pipe_empty(head, tail); +} + +static bool qtfs_pipe_full(struct pipe_inode_info *pipe) +{ + unsigned int head, tail; + + head = READ_ONCE(pipe->head); + tail = READ_ONCE(pipe->tail); + + return pipe_full(head, tail, pipe->max_usage); +} +#endif + +int handle_fifopoll(struct qtserver_arg *arg) +{ + struct qtreq_poll *req = (struct qtreq_poll *)REQ(arg); + struct qtrsp_poll *rsp = (struct qtrsp_poll *)RSP(arg); + struct file *filp = NULL; + struct pipe_inode_info *pipe; + struct inode *inode; + __poll_t mask; + struct poll_wqueues table; + poll_table *pt; + + filp = fget(req->fd); + if (!filp) { + rsp->mask = EPOLLERR; + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_poll); + } + inode = filp->f_inode; + if (!S_ISFIFO(inode->i_mode)) { + msleep(1); + poll_initwait(&table); + pt = &table.pt; + mask = vfs_poll(filp, pt); + poll_freewait(&table); + goto end; + } + pipe = filp->private_data; + if (pipe == NULL) { + qtfs_err("file :%s pipe data is NULL.", filp->f_path.dentry->d_iname); + rsp->ret = QTFS_ERR; + rsp->mask = EPOLLERR; + fput(filp); + return sizeof(struct qtrsp_poll); + } + mask = 0; + if (filp->f_mode & FMODE_READ) { + if (!qtfs_pipe_empty(pipe)) + mask |= EPOLLIN | EPOLLRDNORM; + if (!pipe->writers && filp->f_version != pipe->w_counter) + mask |= EPOLLHUP; + } + + if (filp->f_mode & FMODE_WRITE) { + if (!qtfs_pipe_full(pipe)) + mask |= EPOLLOUT | EPOLLWRNORM; + if (!pipe->readers) + mask |= EPOLLERR; + } +end: + rsp->mask = mask; + rsp->ret = QTFS_OK; + + qtfs_info("handle fifo poll f_mode:%o: %s get poll mask 0x%x\n", + filp->f_mode, filp->f_path.dentry->d_iname, rsp->mask); + fput(filp); + return sizeof(struct qtrsp_poll); +} + +int handle_epollctl(struct qtserver_arg *arg) +{ + struct qtreq_epollctl *req = (struct qtreq_epollctl *)REQ(arg); + struct qtrsp_epollctl *rsp = (struct qtrsp_epollctl *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s*)USERP(arg); + int ret; + struct epoll_event evt; + + evt.data = (__u64)req->event.data; + evt.events = req->event.events; + if (copy_to_user(userp->userp, &evt, sizeof(struct epoll_event))) { + qtfs_err("copy to user failed."); + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_epollctl); + } + ret = qtfs_syscall_epoll_ctl(qtfs_epoll.epfd, req->op, req->fd, userp->userp); + if (ret < 0) { + qtfs_err("handle do epoll ctl failed, ret:%d.", ret); + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_epollctl); + } + qtinfo_cntinc((req->op == EPOLL_CTL_ADD) ? QTINF_EPOLL_ADDFDS : QTINF_EPOLL_DELFDS); + rsp->ret = QTFS_OK; + qtfs_info("handle do epoll ctl success, fd:%d op:%x poll_t:%x.", + req->fd, req->op, (unsigned)req->event.events); + + return sizeof(struct qtrsp_epollctl); +} + +int handle_llseek(struct qtserver_arg *arg) +{ + struct qtreq_llseek *req = (struct qtreq_llseek *)REQ(arg); + struct qtrsp_llseek *rsp = (struct qtrsp_llseek *)RSP(arg); + + qtfs_info("llseek get req fd:%d, off:%lld whence:%d.", req->fd, req->off, req->whence); + rsp->off = qtfs_syscall_lseek(req->fd, req->off, req->whence); + if (rsp->off < 0) { + qtfs_err("llseek ksys lseek return :%ld failed, req fd:%d off:%lld whence:%d.", + rsp->off, req->fd, req->off, req->whence); + rsp->ret = QTFS_ERR; + goto end; + } + rsp->ret = QTFS_OK; +end: + return sizeof(struct qtrsp_llseek); +} + +int handle_exit(struct qtserver_arg *arg) +{ + return 0; +} + +int handle_null(struct qtserver_arg *arg) +{ + qtfs_err("unknown events."); + return 0; +} + +int remotesc_kill(struct qtserver_arg *arg) +{ + struct qtreq_sc_kill *req = (struct qtreq_sc_kill *)REQ(arg); + struct qtrsp_sc_kill *rsp = (struct qtrsp_sc_kill *)RSP(arg); + char tskcomm[TASK_COMM_LEN] = {0}; + struct task_struct *t = qtfs_kern_syms.find_get_task_by_vpid((pid_t)req->pid); + if (req->signum == 0) + goto result; + if (!t) { + qtfs_err("Failed to get task by pid:%d", req->pid); + rsp->ret = -EINVAL; + goto end; + } + get_task_comm(tskcomm, t); + if (!in_white_list_exact(tskcomm, QTFS_WHITELIST_KILL)) { + qtfs_err("Failed to kill pid:%d, comm:%s not in kill white list", req->pid, tskcomm); + rsp->ret = -EPERM; + goto end; + } +result: + rsp->ret = qtfs_syscall_kill(req->pid, req->signum); + qtfs_info("Recv remote kill request, pid:%d signum:%d ret:%ld", req->pid, req->signum, rsp->ret); +end: + return sizeof(struct qtrsp_sc_kill); +} + +int remotesc_sched_getaffinity(struct qtserver_arg *arg) +{ + struct qtreq_sc_sched_affinity *req = (struct qtreq_sc_sched_affinity *)REQ(arg); + struct qtrsp_sc_sched_affinity *rsp = (struct qtrsp_sc_sched_affinity *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s*)USERP(arg); + + if (req->len > AFFINITY_MAX_LEN) { + qtfs_err("invalid len:%lu", req->len); + rsp->ret = -EINVAL; + rsp->len = 0; + goto end; + } + rsp->ret = qtfs_syscall_sched_getaffinity(req->pid, req->len, userp->userp); + if (rsp->ret < 0) { + qtfs_err("get affinity failed ret:%ld", rsp->ret); + rsp->len = 0; + goto end; + } + if (copy_from_user(rsp->user_mask_ptr, userp->userp, req->len)) { + qtfs_err("copy affinity failed"); + rsp->len = 0; + rsp->ret = -EFAULT; + goto end; + } + rsp->len = req->len; + qtfs_info("pid:%d get affinity successed", req->pid); + return sizeof(struct qtrsp_sc_sched_affinity) + rsp->len * sizeof(unsigned long); + +end: + return sizeof(struct qtrsp_sc_sched_affinity); +} + +int remotesc_sched_setaffinity(struct qtserver_arg *arg) +{ + struct qtreq_sc_sched_affinity *req = (struct qtreq_sc_sched_affinity *)REQ(arg); + struct qtrsp_sc_sched_affinity *rsp = (struct qtrsp_sc_sched_affinity *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s*)USERP(arg); + + if (req->len > AFFINITY_MAX_LEN || req->len < 0) { + qtfs_err("invalid len:%lu", req->len); + rsp->ret = -EINVAL; + rsp->len = 0; + goto end; + } + if (copy_to_user(userp->userp, req->user_mask_ptr, req->len)) { + qtfs_err("copy to user failed len:%lu", req->len); + rsp->ret = -EFAULT; + rsp->len = 0; + goto end; + } + rsp->ret = qtfs_syscall_sched_setaffinity(req->pid, req->len, userp->userp); + if (rsp->ret < 0) { + qtfs_err("set affinity failed, ret:%ld pid:%d len:%lu", rsp->ret, req->pid, req->len); + goto end; + } + qtfs_info("set affinity successed mask:%lx%lx", req->user_mask_ptr[0], req->user_mask_ptr[1]); +end: + return sizeof(struct qtrsp_sc_sched_affinity); +} + +static struct qtserver_ops qtfs_server_handles[] = { + {QTFS_REQ_NULL, req_check_none, handle_null, "null"}, + {QTFS_REQ_MOUNT, req_check_mount, handle_mount, "mount"}, + {QTFS_REQ_OPEN, req_check_open, handle_open, "open"}, + {QTFS_REQ_CLOSE, req_check_close, handle_close, "close"}, + {QTFS_REQ_READ, req_check_none, handle_null, "read"}, + {QTFS_REQ_READITER, req_check_readiter, handle_readiter, "readiter"}, + {QTFS_REQ_WRITE, req_check_write, handle_write, "write"}, + {QTFS_REQ_LOOKUP, req_check_lookup, handle_lookup, "lookup"}, + {QTFS_REQ_READDIR, req_check_readdir, handle_readdir, "readdir"}, + {QTFS_REQ_MKDIR, req_check_mkdir, handle_mkdir, "mkdir"}, + {QTFS_REQ_RMDIR, req_check_rmdir, handle_rmdir, "rmdir"}, + {QTFS_REQ_GETATTR, req_check_getattr, handle_getattr, "getattr"}, + {QTFS_REQ_SETATTR, req_check_setattr, handle_setattr, "setattr"}, + {QTFS_REQ_ICREATE, req_check_icreate, handle_icreate, "icreate"}, + {QTFS_REQ_MKNOD, req_check_mknod, handle_mknod, "mknod"}, + {QTFS_REQ_UNLINK, req_check_unlink, handle_unlink, "unlink"}, + {QTFS_REQ_SYMLINK, req_check_symlink, handle_symlink, "symlink"}, + {QTFS_REQ_LINK, req_check_link, handle_link, "link"}, + {QTFS_REQ_GETLINK, req_check_getlink, handle_getlink, "getlink"}, + {QTFS_REQ_READLINK, req_check_none, handle_null, "readlink"}, + {QTFS_REQ_RENAME, req_check_rename, handle_rename, "rename"}, + + {QTFS_REQ_XATTRLIST, req_check_xattrlist, handle_xattrlist, "xattrlist"}, + {QTFS_REQ_XATTRGET, req_check_xattrget, handle_xattrget, "xattrget"}, + {QTFS_REQ_XATTRSET, req_check_xattrset, handle_xattrset, "xattrset"}, + + {QTFS_REQ_SYSMOUNT, req_check_sysmount, handle_syscall_mount, "sysmount"}, + {QTFS_REQ_SYSUMOUNT, req_check_sysumount, handle_syscall_umount, "sysumount"}, + {QTFS_REQ_FIFOPOLL, req_check_fifopoll, handle_fifopoll, "fifo_poll"}, + + {QTFS_REQ_STATFS, req_check_statfs, handle_statfs, "statfs"}, + {QTFS_REQ_IOCTL, req_check_ioctl, handle_ioctl, "ioctl"}, + + {QTFS_REQ_EPOLL_CTL, req_check_epoll_ctl, handle_epollctl, "epollctl"}, + {QTFS_REQ_EPOLL_EVENT, req_check_none, NULL, "epollevent"}, + + {QTFS_REQ_LLSEEK, req_check_llseek, handle_llseek, "llseek"}, + + // remote syscall or capability + {QTFS_SC_KILL, req_check_sc_kill, remotesc_kill, "remotesc_kill"}, + {QTFS_SC_SCHED_GETAFFINITY, req_check_sc_sched_getaffinity, remotesc_sched_getaffinity, "sched_getaffinity"}, + {QTFS_SC_SCHED_SETAFFINITY, req_check_sc_sched_setaffinity, remotesc_sched_setaffinity, "sched_setaffinity"}, + + {QTFS_REQ_EXIT, req_check_none, handle_exit, "exit"}, // keep this handle at the end +}; + +int qtfs_conn_server_run(struct qtfs_conn_var_s *pvar) +{ + int ret; + struct qtreq *req; + struct qtreq *rsp; + unsigned long totalproc = 0; + + req = pvar->vec_recv.iov_base; + rsp = pvar->vec_send.iov_base; + do { + ret = qtfs_conn_recv_block(pvar); + if (ret == -EPIPE) { + qtfs_err("qtfs server thread recv EPIPE, restart the connection."); + qtfs_sm_reconnect(pvar); + break; + } + if (ret < 0) + break; + if (req->type >= QTFS_REQ_INV) { + qtfs_err("qtfs server recv unknown operate type:%d\n", req->type); + rsp->type = req->type; + rsp->len = 0; + rsp->err = QTFS_ERR; + } else { + struct qtserver_arg arg; + arg.data = req->data; + arg.out = rsp->data; + if (qtfs_server_handles[req->type].precheck((void *)req->data) == QTFS_CHECK_ERR) { + rsp->type = req->type; + rsp->len = 0; + rsp->err = QTFS_ERR; + qtinfo_reqcheckinc(req->type); + qtfs_err("qtfs server req type:%u precheck failed.", req->type); + goto out; + } + read_lock(&g_userp_rwlock); + arg.userp = &qtfs_userps[pvar->cur_threadidx]; + if (arg.userp->userp == NULL || arg.userp->userp2 == NULL) + qtfs_err("server run userp or userp2 is invalid"); + rsp->len = qtfs_server_handles[req->type].handle(&arg); + read_unlock(&g_userp_rwlock); + rsp->type = req->type; + rsp->err = QTFS_OK; + totalproc++; + qtinfo_recvinc(req->type); + } + if (rsp->len > QTFS_REQ_MAX_LEN) { + qtfs_crit("handle rsp len error type:%d len:%lu", rsp->type, rsp->len); + WARN_ON(1); + rsp->len = QTFS_REQ_MAX_LEN - 1; + rsp->err = QTFS_ERR; + } +out: + rsp->seq_num = req->seq_num; + pvar->vec_send.iov_len = QTFS_MSG_HEAD_LEN + rsp->len; + qtfs_debug("Server thread:%d count:%lu recv len:%d type:%d(%s) seq_num:%lu, reqlen:%lu, resp len:%lu, rsp threadidx:%d.\n", + pvar->cur_threadidx, totalproc, ret, req->type, (req->type >= QTFS_REQ_INV) ? "null" : qtfs_server_handles[req->type].str, + req->seq_num, req->len, pvar->vec_send.iov_len, pvar->cur_threadidx); + ret = qtfs_conn_send(pvar); + if (ret == -EPIPE) { + qtfs_err("qtfs server send get EPIPE, just restart the connection\n"); + qtfs_sm_reconnect(pvar); + break; + } + if (ret < 0) { + qtfs_err("conn send failed, ret:%d\n", ret); + WARN_ON(1); + } + qtinfo_sendinc(rsp->type); + } while(0); + + return (ret < 0) ? QTERROR : QTOK; +} + diff --git a/qtfs/qtfs_server/fsops.h b/qtfs/qtfs_server/fsops.h new file mode 100644 index 0000000..79f8264 --- /dev/null +++ b/qtfs/qtfs_server/fsops.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __QTFS_SERVER_FSOPS_H__ +#define __QTFS_SERVER_FSOPS_H__ + +struct qtfs_getdents { + struct dir_context ctx; + int vldcnt; + struct qtfs_dirent64 * dir; + int prev_reclen; + int count; +}; + +#endif diff --git a/qtfs/qtfs_server/qtfs-server.c b/qtfs/qtfs_server/qtfs-server.c new file mode 100644 index 0000000..12d2ed0 --- /dev/null +++ b/qtfs/qtfs_server/qtfs-server.c @@ -0,0 +1,440 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include + +#include "conn.h" +#include "qtfs-server.h" +#include "comm.h" +#include "log.h" +#include "req.h" +#include "symbol_wrapper.h" + +#define QTFS_EPOLL_TIMEO 1000 // unit ms +#define QTFS_EPOLL_RETRY_INTVL 500 // unit ms + +int qtfs_server_thread_run = 1; +DEFINE_RWLOCK(g_userp_rwlock); +struct qtserver_fd_bitmap qtfs_fd_bitmap; + +long qtfs_server_misc_ioctl(struct file *file, unsigned int cmd, unsigned long arg); + +struct file_operations qtfs_misc_fops = { + .owner=THIS_MODULE, + .unlocked_ioctl = qtfs_server_misc_ioctl, +}; + +struct qtfs_server_epoll_s qtfs_epoll = { + .epfd = -1, + .event_nums = 0, + .events = NULL, + .kevents = NULL, +}; +rwlock_t qtfs_epoll_rwlock; + +void qtfs_server_epoll_exit(void) +{ + write_lock(&qtfs_epoll_rwlock); + if (qtfs_epoll.kevents == NULL) { + write_unlock(&qtfs_epoll_rwlock); + return; + } + kfree(qtfs_epoll.kevents); + qtfs_epoll.kevents = NULL; + write_unlock(&qtfs_epoll_rwlock); + return; +} + +long qtfs_server_epoll_thread(struct qtfs_conn_var_s *pvar) +{ + int n; + struct qtreq_epollevt *req; + struct qtrsp_epollevt *rsp; + struct qtreq *head; + int sendlen; + int ret = 0; + int i = 0; + + if (qtfs_epoll.epfd == -1) { + qtfs_err("qtfs epoll wait error, epfd is invalid."); + return QTERROR; + } + if (false == pvar->conn_ops->conn_connected(&pvar->conn_var)) { + qtfs_warn("qtfs epoll thread disconnected, now try to reconnect."); + ret = qtfs_sm_reconnect(pvar); + } else { + ret = qtfs_sm_active(pvar); + } + if (ret != QTOK) { + qtfs_err("qtfs epoll thread connect state error, can't work."); + msleep(QTFS_EPOLL_RETRY_INTVL); + return QTERROR; + } + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + rsp = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_RECV); + head = pvar->vec_send.iov_base; + do { + n = qtfs_syscall_epoll_wait(qtfs_epoll.epfd, qtfs_epoll.events, qtfs_epoll.event_nums, 0); + if (n == 0) { + msleep(1); + break; + } + if (n < 0 || n > QTFS_MAX_EPEVENTS_NUM) { + msleep(QTFS_EPOLL_RETRY_INTVL); + qtfs_err("epoll get new events number failed:%d ", n); + break; + } + qtfs_info(">>epoll get new events number:%d.", n); + if (copy_from_user(qtfs_epoll.kevents, qtfs_epoll.events, sizeof(struct epoll_event) * n)) { + qtfs_err("qtfs copy epoll events failed, events lost."); + WARN_ON(1); + break; + } + for (i = 0; i < n; i++) { + req->events[i].data = qtfs_epoll.kevents[i].data; + req->events[i].events = qtfs_epoll.kevents[i].events; + qtfs_info("epoll thread head req:%lx.", (unsigned long)req); + } + req->event_nums = n; + sendlen = sizeof(struct qtreq_epollevt) - sizeof(req->events) + n * sizeof(struct qtreq_epoll_event); + if (sendlen > QTFS_REQ_MAX_LEN) { + qtfs_err("qtfs epoll events size(%d) larger than qtfs message size.", sendlen); + WARN_ON(1); + break; + } + pvar->vec_send.iov_len = QTFS_MSG_HEAD_LEN + sendlen; + head->len = sendlen; + head->type = QTFS_REQ_EPOLL_EVENT; + ret = qtfs_conn_send(pvar); + qtfs_info("qtfs send msg conn: sendlen:%lu ret:%d.", + (unsigned long)pvar->vec_send.iov_len, ret); + if (ret == -EPIPE) { + qtfs_err("epoll wait send events failed get EPIPE, just wait new connection."); + qtfs_sm_reconnect(pvar); + break; + } + if (ret < 0) { + qtfs_err("epoll wait send events failed, ret:%d.", ret); + WARN_ON(1); + } +retry: + ret = qtfs_conn_recv_block(pvar); + if (ret == -EAGAIN) { + if (qtfs_server_thread_run == 0) { + qtfs_warn("qtfs module exiting, goodbye!"); + return QTEXIT; + } + goto retry; + } + if (ret == -EPIPE) { + qtfs_err("epoll recv events failed get EPIPE, just wait new connection."); + qtfs_sm_reconnect(pvar); + break; + } + }while (0); + + return (ret < 0) ? QTERROR : QTOK; +} + +long qtfs_server_epoll_init(void) +{ + struct qtfs_conn_var_s *pvar = NULL; + struct qtreq_epollevt *req; + + pvar = qtfs_epoll_establish_conn(); + if (pvar == NULL) { + return QTERROR; + } + qtfs_epoll_var = pvar; + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + qtfs_info("qtfs epoll events req size:%lu, events size:%lu, struct:%lu.", + sizeof(struct qtreq_epollevt), sizeof(req->events), + sizeof(struct qtreq_epoll_event)); + qtfs_info("qtfs epoll wait thread, epfd:%d nums:%d.", + qtfs_epoll.epfd, qtfs_epoll.event_nums); + + return QTOK; +} + +int qtfs_server_fd_bitmap_init(void) +{ + struct rlimit fd_rlim; + if (qtfs_fd_bitmap.bitmap != NULL) { + qtfs_info("free old bitmap"); + kfree(qtfs_fd_bitmap.bitmap); + qtfs_fd_bitmap.bitmap = NULL; + qtfs_fd_bitmap.nbits = 0; + } + // fd_rlim is get from current task struct, can be trusted + fd_rlim = current->signal->rlim[RLIMIT_NOFILE]; + qtfs_info("task rlimit cur:%lu max:%lu", fd_rlim.rlim_cur, fd_rlim.rlim_max); + qtfs_fd_bitmap.bitmap = (unsigned long *)kmalloc(BITS_TO_BYTES(fd_rlim.rlim_cur), GFP_KERNEL); + if (qtfs_fd_bitmap.bitmap == NULL) { + qtfs_err("kmalloc len:%lu failed.", BITS_TO_BYTES(fd_rlim.rlim_cur)); + return -1; + } + qtfs_fd_bitmap.nbits = fd_rlim.rlim_cur; + bitmap_zero(qtfs_fd_bitmap.bitmap, qtfs_fd_bitmap.nbits); + return 0; +} + +long qtfs_server_misc_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + int i; + long ret = 0; + struct qtfs_conn_var_s *pvar; + struct qtfs_thread_init_s init_userp; + switch (cmd) { + case QTFS_IOCTL_THREAD_INIT: + pvar = qtfs_conn_get_param_errcode(); + if (IS_ERR_OR_NULL(pvar)) { + qtfs_err("init pvar get failed, pvar:%ld", (long)pvar); + if (PTR_ERR(pvar) == -EADDRINUSE) + return EADDRINUSE; + } else { + qtfs_conn_put_param(pvar); + } + if (!write_trylock(&g_userp_rwlock)) { + qtfs_err("try lock userps failed."); + return QTERROR; + } + if (qtfs_server_fd_bitmap_init() < 0) { + qtfs_err("fd bitmap init failed."); + write_unlock(&g_userp_rwlock); + return QTERROR; + } + if (copy_from_user(&init_userp, (void __user *)arg, sizeof(struct qtfs_thread_init_s))) { + qtfs_err("qtfs ioctl thread init copy from user failed."); + write_unlock(&g_userp_rwlock); + return QTERROR; + } + if (qtfs_userps == NULL || init_userp.thread_nums > QTFS_MAX_THREADS || init_userp.thread_nums == 0) { + qtfs_err("qtfs ioctl thread init userps invalid thread nums:%d.", init_userp.thread_nums); + write_unlock(&g_userp_rwlock); + return QTERROR; + } + memset(qtfs_userps, 0, QTFS_MAX_THREADS * sizeof(struct qtfs_server_userp_s)); + if (init_userp.thread_nums > QTFS_MAX_THREADS) { + qtfs_err("qtfs ioctl thread init invalid input thread_num:%d", init_userp.thread_nums); + write_unlock(&g_userp_rwlock); + return QTERROR; + } + if (copy_from_user(qtfs_userps, (void __user *)init_userp.userp, + init_userp.thread_nums * sizeof(struct qtfs_server_userp_s))) { + qtfs_err("qtfs ioctl thread init copy from userp failed."); + write_unlock(&g_userp_rwlock); + return QTERROR; + } + for (i = 0; i < init_userp.thread_nums; i++) { + if (qtfs_userps[i].size > QTFS_USERP_MAXSIZE || + !access_ok(qtfs_userps[i].userp, qtfs_userps[i].size) || + !access_ok(qtfs_userps[i].userp2, qtfs_userps[i].size)) { + qtfs_err("userp set failed"); + ret = QTERROR; + write_unlock(&g_userp_rwlock); + break; + } + qtfs_info("userp set idx:%d size:%lu", i, qtfs_userps[i].size); + } + write_unlock(&g_userp_rwlock); + break; + case QTFS_IOCTL_THREAD_RUN: + pvar = qtfs_conn_get_param(); + if (pvar == NULL) + return QTERROR; + ret = qtfs_conn_server_run(pvar); + if (ret == QTEXIT) { + qtfs_warn("qtfs thread idx:%d exit.", pvar->cur_threadidx); + qtfs_sm_exit(pvar); + qtinfo_cntdec(QTINF_ACTIV_CONN); + } + qtfs_conn_put_param(pvar); + break; + case QTFS_IOCTL_EPFDSET: + write_lock(&qtfs_epoll_rwlock); + if (qtfs_epoll.kevents != NULL) { + kfree(qtfs_epoll.kevents); + qtfs_epoll.kevents = NULL; + } + if (copy_from_user(&qtfs_epoll, (void __user *)arg, sizeof(struct qtfs_server_epoll_s))) { + qtfs_err("copy epoll struct from arg failed."); + ret = QTERROR; + write_unlock(&qtfs_epoll_rwlock); + break; + } + if (qtfs_epoll.event_nums > QTFS_MAX_EPEVENTS_NUM || qtfs_epoll.event_nums == 0) { + qtfs_err("epoll arg set failed, event nums:%d too big", qtfs_epoll.event_nums); + ret = QTERROR; + write_unlock(&qtfs_epoll_rwlock); + break; + } + if (qtfs_epoll.epfd < 3) { + qtfs_err("epoll epfd set failed, epfd:%d should be greater than 2", qtfs_epoll.epfd); + ret = QTERROR; + write_unlock(&qtfs_epoll_rwlock); + break; + } + if (!access_ok(qtfs_epoll.events, qtfs_epoll.event_nums * sizeof(struct epoll_event))) { + qtfs_err("epoll events set failed, check pointer of qtfs_epoll.events failed"); + ret = QTERROR; + write_unlock(&qtfs_epoll_rwlock); + break; + } + qtfs_info("epoll arg set, epfd:%d event nums:%d events.", + qtfs_epoll.epfd, qtfs_epoll.event_nums); + qtfs_epoll.kevents = (struct epoll_event *)kmalloc(sizeof(struct epoll_event) * + qtfs_epoll.event_nums, GFP_KERNEL); + if (qtfs_epoll.kevents == NULL) { + qtfs_err("epoll kernel events kmalloc failed."); + ret = QTERROR; + write_unlock(&qtfs_epoll_rwlock); + break; + } + write_unlock(&qtfs_epoll_rwlock); + break; + case QTFS_IOCTL_EPOLL_THREAD_INIT: + write_lock(&qtfs_epoll_rwlock); + ret = qtfs_server_epoll_init(); + write_unlock(&qtfs_epoll_rwlock); + break; + case QTFS_IOCTL_EPOLL_THREAD_RUN: + write_lock(&qtfs_epoll_rwlock); + if (qtfs_epoll_var == NULL) { + qtfs_err("qtfs epoll thread run failed, var is invalid."); + ret = QTERROR; + write_unlock(&qtfs_epoll_rwlock); + break; + } + ret = qtfs_server_epoll_thread(qtfs_epoll_var); + write_unlock(&qtfs_epoll_rwlock); + if (ret == QTEXIT) { + qtfs_info("qtfs epoll thread exit."); + qtfs_epoll_cut_conn(qtfs_epoll_var); + } + break; + case QTFS_IOCTL_EXIT: + if (arg != 0 && arg != 1) { + qtfs_err("qtfs exit input invalid, should be 0 or 1."); + ret = QTERROR; + break; + } + qtfs_whitelist_clearall(); + qtfs_info("qtfs server threads run set to:%lu.", arg); + qtfs_server_thread_run = arg; + break; + case QTFS_IOCTL_ALLINFO: + case QTFS_IOCTL_CLEARALL: + case QTFS_IOCTL_LOGLEVEL: + case QTFS_IOCTL_WL_ADD: + case QTFS_IOCTL_WL_DEL: + case QTFS_IOCTL_WL_GET: + ret = qtfs_misc_ioctl(file, cmd, arg); + break; + default: + qtfs_err("qtfs misc ioctl unknown cmd:%u.", cmd); + ret = QTERROR; + break; + } + + return ret; +} + +static int __init qtfs_server_init(void) +{ + qtfs_log_init(qtfs_log_level, sizeof(qtfs_log_level)); + if (qtfs_kallsyms_hack_init() != 0) + return -1; + rwlock_init(&qtfs_epoll_rwlock); + qtfs_whitelist_initset(); + + qtfs_diag_info = (struct qtinfo *)kmalloc(sizeof(struct qtinfo), GFP_KERNEL); + if (qtfs_diag_info == NULL) { + qtfs_err("kmalloc qtfs diag info failed."); + return -1; + } + memset(qtfs_diag_info, 0, sizeof(struct qtinfo)); + qtfs_userps = (struct qtfs_server_userp_s *)kmalloc( + QTFS_MAX_THREADS * sizeof(struct qtfs_server_userp_s), GFP_KERNEL); + if (qtfs_userps == NULL) { + qtfs_err("kmalloc qtfs userps failed, nums:%d", QTFS_MAX_THREADS); + kfree(qtfs_diag_info); + return -1; + } + memset(qtfs_userps, 0, QTFS_MAX_THREADS * sizeof(struct qtfs_server_userp_s)); + qtfs_fd_bitmap.bitmap = NULL; + qtfs_fd_bitmap.nbits = 0; + qtfs_conn_param_init(); + if (qtfs_syscall_replace_start()) { + qtfs_err("qtfs syscall replace failed."); + goto err_syscall; + } + if (qtfs_misc_register()) { + qtfs_err("qtfs misc device register failed."); + goto err_misc; + } + return 0; +err_misc: + qtfs_syscall_replace_stop(); +err_syscall: + kfree(qtfs_userps); + kfree(qtfs_diag_info); + return -1; +} + +static void __exit qtfs_server_exit(void) +{ + qtfs_mod_exiting = true; + qtfs_server_thread_run = 0; + + qtfs_conn_param_fini(); + + if (qtfs_epoll_var != NULL) { + qtfs_epoll_cut_conn(qtfs_epoll_var); + qtfs_conn_fini(qtfs_epoll_var); + qtfs_epoll_var->conn_ops->conn_var_fini(qtfs_epoll_var); + kfree(qtfs_epoll_var); + qtfs_epoll_var = NULL; + } + if (qtfs_diag_info != NULL) { + kfree(qtfs_diag_info); + qtfs_diag_info = NULL; + } + write_lock(&g_userp_rwlock); + if (qtfs_userps != NULL) { + kfree(qtfs_userps); + qtfs_userps = NULL; + } + if (qtfs_fd_bitmap.bitmap != NULL) { + kfree(qtfs_fd_bitmap.bitmap); + qtfs_fd_bitmap.bitmap = NULL; + qtfs_fd_bitmap.nbits = 0; + } + write_unlock(&g_userp_rwlock); + qtfs_whitelist_exit(); + qtfs_server_epoll_exit(); + + qtfs_misc_destroy(); + qtfs_syscall_replace_stop(); + qtfs_info("qtfs server exit done.\n"); + return; +} + +module_init(qtfs_server_init); +module_exit(qtfs_server_exit); +MODULE_AUTHOR("liqiang64@huawei.com"); +MODULE_LICENSE("GPL"); diff --git a/qtfs/qtfs_server/qtfs-server.h b/qtfs/qtfs_server/qtfs-server.h new file mode 100644 index 0000000..8135dbb --- /dev/null +++ b/qtfs/qtfs_server/qtfs-server.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __QTFS_SERVER_H__ +#define __QTFS_SERVER_H__ + +extern int qtfs_server_thread_run; +extern struct qtfs_server_epoll_s qtfs_epoll; +extern int qtfs_mod_exiting; +extern rwlock_t g_userp_rwlock; +extern struct qtserver_fd_bitmap qtfs_fd_bitmap; + +struct qtserver_arg { + char *data; + char *out; + struct qtfs_server_userp_s *userp; +}; + +struct qtserver_ops { + int type; + int (*precheck) (void *); // check req from socket recv + // return int is output len. + int (*handle) (struct qtserver_arg *); + char str[32]; +}; + +struct qtserver_fd_bitmap { + unsigned int nbits; + unsigned long *bitmap; +}; + +int qtfs_conn_server_run(struct qtfs_conn_var_s *pvar); +long qtfs_misc_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +int qtfs_misc_register(void); +void qtfs_misc_destroy(void); +void qtfs_whitelist_exit(void); +long qtfs_misc_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +void qtfs_whitelist_clearall(void); +void qtfs_whitelist_initset(void); + +#endif diff --git a/qtfs/qtfs_server/qtfs_fifo_server/Cargo.toml b/qtfs/qtfs_server/qtfs_fifo_server/Cargo.toml new file mode 100644 index 0000000..803b5ed --- /dev/null +++ b/qtfs/qtfs_server/qtfs_fifo_server/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "qtfs_fifo_server" +version = "1.0.0" +edition = "2021" + +[dependencies] +tokio = { version = "1.29.1", features = ["full"]} +libc = "0.2" +rlimit = "0.10.1" \ No newline at end of file diff --git a/qtfs/qtfs_server/qtfs_fifo_server/src/cofifo.rs b/qtfs/qtfs_server/qtfs_fifo_server/src/cofifo.rs new file mode 100644 index 0000000..24c305d --- /dev/null +++ b/qtfs/qtfs_server/qtfs_fifo_server/src/cofifo.rs @@ -0,0 +1,337 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * qtfs licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Liqiang + * Create: 2023-07-26 + * Description: + *******************************************************************************/ + +use tokio::net::TcpStream; +use std::mem; +use tokio::fs::File; +use tokio::fs; +use std::os::unix::fs::FileTypeExt; +use tokio::fs::OpenOptions; + +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +struct Qtreq { + // magic: [u8; 4], //magic: 0x5aa55aa5 + msgtype: u32, + error: u32, + seq_num: u64, + len: usize, +} + +const QTFS_REQ_OPEN: u32 = 2; +const QTFS_REQ_CLOSE: u32 = 3; +const QTFS_REQ_READ: u32 = 5; +const QTFS_REQ_WRITE: u32 = 6; +pub async fn qtfs_fifo_server(stream: TcpStream, idx: usize) { + let mut conn = Conn {stream}; + let mut head: Qtreq; + + match conn.qtfs_req_head(idx.clone()).await { + Ok(h) => head = h, + Err(e) => { + println!("Recv invalid head exit this proc :{}.", e); + return; + } + } + if head.msgtype != QTFS_REQ_OPEN { + println!("first msg type is invalid"); + return; + } + let file = match conn.qtfs_fifo_open(head.clone()).await { + Ok(f) => { + head.len = mem::size_of::(); + conn.req_head_ack(head).await; + conn.open_ack(0).await; + f + } + Err(e) => { + head.len = mem::size_of::(); + println!("Open fifo error:{}", e); + conn.req_head_ack(head).await; + conn.open_ack(1).await; + return; + } + }; + + 'main: loop { + let mut head: Qtreq; + match conn.qtfs_req_head(idx.clone()).await { + Ok(h) => head = h, + Err(e) => { + println!("head recv failed, {}.", e); + return; + } + } + match head.msgtype { + QTFS_REQ_OPEN => { + println!("Fifo is opened and recv open request again!"); + head.len = mem::size_of::(); + conn.req_head_ack(head).await; + conn.open_ack(1).await; + } + QTFS_REQ_CLOSE => { + println!("Close req idx:{}", idx.clone()); + head.len = 0; + conn.req_head_ack(head).await; + break 'main; + } + QTFS_REQ_READ => { + println!("Read req idx:{}", idx.clone()); + conn.qtfs_fifo_read(file.try_clone().await.unwrap(), head).await; + } + QTFS_REQ_WRITE => { + println!("Write req idx:{}", idx.clone()); + conn.qtfs_fifo_write(file.try_clone().await.unwrap(), head).await; + } + _ => { + println!("Recv invalid msg type"); + } + } + } + println!("Fifo server idx:{} is closed.", idx); +} + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +struct Qtreqopen { + flags: u64, + mode: u32, +} +#[repr(C, packed)] +struct Qtrspopen { + ret: i32, +} + +#[repr(C, packed)] +struct Qtreqread { + len: u64, +} + +#[repr(C, packed)] +struct Qtrspread { + errno: i32, + len: u64, +} + + + +#[repr(C, packed)] +struct Qtreqwrite { + len: u64, +} +#[repr(C, packed)] +struct Qtrspwrite { + errno: i32, + len: u64, +} + +struct Conn { + stream: TcpStream, +} + +impl Conn { + // sync head magic bytes sequence: 0x5a 0xa5 0x5a 0xa5 + // 逐字节读取magic，连续匹配的四个字节即视为同步包头 + async fn package_sync(&mut self) { + let mut byte: [u8; 1] = [0; 1]; + loop { + self.stream.read_exact(&mut byte).await.unwrap(); + if byte[0] != 0x5a {continue;} + self.stream.read_exact(&mut byte).await.unwrap(); + if byte[0] != 0xa5 {continue;} + self.stream.read_exact(&mut byte).await.unwrap(); + if byte[0] != 0x5a {continue;} + self.stream.read_exact(&mut byte).await.unwrap(); + if byte[0] != 0xa5 {continue;} + break; + } + } + + async fn send_magic_head(&mut self) { + const MAGIC: [u8; 4] = [0x5a, 0xa5, 0x5a, 0xa5]; + let _ = self.stream.write_all(&MAGIC[0..4]).await; + } + + async fn qtfs_req_head(&mut self, _idx: usize) -> Result { + const HEADSIZE: usize = mem::size_of::(); + self.package_sync().await; + let mut msghead = [0; HEADSIZE]; + self.stream.read_exact(&mut msghead).await?; + let head = Qtreq { + msgtype: u32::from_le_bytes(msghead[0..4].try_into().unwrap()), + error: u32::from_le_bytes(msghead[4..8].try_into().unwrap()), + seq_num: u64::from_le_bytes(msghead[8..16].try_into().unwrap()), + len: usize::from_le_bytes(msghead[16..16+mem::size_of::()].try_into().unwrap()), + }; + let reqtype: String = match head.msgtype { + QTFS_REQ_OPEN => String::from("Open"), + QTFS_REQ_CLOSE => String::from("Close"), + QTFS_REQ_READ => String::from("Read"), + QTFS_REQ_WRITE => String::from("Write"), + _ => String::from("Unknown"), + }; + println!("Recv new head type:{} msg:{:?}", reqtype, head); + Ok(head) + } + + async fn qtfs_fifo_open(&mut self, head: Qtreq) -> Result { + const HEADSIZE: usize = mem::size_of::(); + let mut openhead = [0; HEADSIZE]; + + if head.len >= 4096 + HEADSIZE { + println!("qtfs fifo len invalid"); + return Err(1); + } + self.stream.read_exact(&mut openhead).await.unwrap(); + let openhead1 = Qtreqopen { + flags: u64::from_le_bytes(openhead[0..8].try_into().unwrap()), + mode: u32::from_le_bytes(openhead[8..12].try_into().unwrap()), + }; + println!("open head:{:?}", openhead1); + let mut path = Vec::with_capacity(head.len - HEADSIZE); + path.resize(head.len - HEADSIZE, 0); + self.stream.read_exact(&mut path).await.unwrap(); + + let getstr = String::from_utf8(path).unwrap(); + let pathstr = getstr.trim_end_matches('\0').trim(); + match fs::metadata(pathstr.clone()).await { + Ok(meta) => { + if meta.file_type().is_fifo() == false { + println!("Requst path:{} not fifo!", pathstr); + return Err(1); + } + } + Err(_) => { + println!("path:{} check failed.", pathstr); + return Err(1); + } + }; + println!("Recv open path:{}", pathstr); + let file = OpenOptions::new() + .read(true) + .write(true) + //.custom_flags(libc::O_NONBLOCK) + .open(pathstr).await.unwrap(); + + Ok(file) + } + + async fn qtfs_fifo_read(&mut self, mut file: File, mut reqhead: Qtreq) { + let mut head = [0; mem::size_of::()]; + self.stream.read_exact(&mut head).await.unwrap(); + let req = Qtreqread { + len: u64::from_le_bytes(head[0..8].try_into().unwrap()), + }; + let len = std::cmp::min(req.len, 4096); + + let mut rsp = Qtrspread { + errno: 0, + len: 0, + }; + + let mut buf = Vec::with_capacity(len.try_into().unwrap()); + buf.resize(len.try_into().unwrap(), 0); + + match file.read(&mut buf).await { + Ok(n) => { + rsp.len = n as u64; + let send = unsafe { + let ptr = &rsp as *const Qtrspread as *const u8; + std::slice::from_raw_parts(ptr, mem::size_of::()) + }; + println!("Read {} bytes from fifo", n.clone()); + reqhead.len = mem::size_of::(); + self.req_head_ack(reqhead).await; + self.stream.write_all(&send[..mem::size_of::()]).await.unwrap(); + let _ = self.stream.write_all(&buf[..n]).await.unwrap(); + } + Err(e) => { + rsp.errno = -1; + rsp.len = 0; + let send = unsafe { + let ptr = &rsp as *const Qtrspread as *const u8; + std::slice::from_raw_parts(ptr, mem::size_of::()) + }; + reqhead.len = mem::size_of::(); + self.req_head_ack(reqhead).await; + self.stream.write_all(&send[..mem::size_of::()]).await.unwrap(); + println!("Read from fifo error:{}", e); + } + } + } + + async fn qtfs_fifo_write(&mut self, mut file: File, mut reqhead: Qtreq) { + let mut whead = [0; mem::size_of::()]; + self.stream.read_exact(&mut whead).await.unwrap(); + let len = u64::from_le_bytes(whead[0..8].try_into().unwrap()); + + // 最大接收一次性写入4k + let len = std::cmp::min(len, 4096); + + let mut rsp = Qtrspwrite { + errno: 0, + len: 0, + }; + + let mut buf = Vec::with_capacity(len.try_into().unwrap()); + buf.resize(len.try_into().unwrap(), 0); + self.stream.read_exact(&mut buf).await.unwrap(); + + match file.write_all(&mut buf[..len as usize]).await { + Ok(_) => { + rsp.len = len as u64; + reqhead.len = mem::size_of::(); + self.req_head_ack(reqhead).await; + self.write_ack(rsp).await; + println!("Write fifo ok, send ack."); + } + Err(e) => { + rsp.len = 0; + reqhead.len = mem::size_of::(); + self.req_head_ack(reqhead).await; + self.write_ack(rsp).await; + println!("Write failed {}.", e); + } + } + } + + async fn req_head_ack(&mut self, head: Qtreq) { + let send = unsafe { + let ptr = &head as *const Qtreq as *const u8; + std::slice::from_raw_parts(ptr, mem::size_of::()) + }; + self.send_magic_head().await; + self.stream.write_all(&send[..mem::size_of::()]).await.expect("req head ack failed"); + } + + async fn open_ack(&mut self, retcode: i32) { + let rsp = Qtrspopen {ret: retcode,}; + let send = unsafe { + let ptr = &rsp as *const Qtrspopen as *const u8; + std::slice::from_raw_parts(ptr, mem::size_of::()) + }; + self.stream.write_all(&send[..mem::size_of::()]).await.expect("Response open failed"); + } + + async fn write_ack(&mut self, rsp: Qtrspwrite) { + let send = unsafe { + let ptr = &rsp as *const Qtrspwrite as *const u8; + std::slice::from_raw_parts(ptr, mem::size_of::()) + }; + self.stream.write_all(&send[..mem::size_of::()]).await.expect("Response write failed"); + } +} \ No newline at end of file diff --git a/qtfs/qtfs_server/qtfs_fifo_server/src/main.rs b/qtfs/qtfs_server/qtfs_fifo_server/src/main.rs new file mode 100644 index 0000000..4f8c0c4 --- /dev/null +++ b/qtfs/qtfs_server/qtfs_fifo_server/src/main.rs @@ -0,0 +1,72 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * qtfs licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Liqiang + * Create: 2023-07-26 + * Description: + *******************************************************************************/ + +use std::env; +use std::net::TcpListener; +use tokio::net::TcpListener as AsyncTcpListener; +use tokio::runtime::Builder; + +extern crate rlimit; +use rlimit::Resource; +use rlimit::setrlimit; +mod cofifo; + +async fn set_rlimit_fd(){ + let rlimit = Resource::NOFILE; + let fd_limit = 65536; + match setrlimit(rlimit, fd_limit, fd_limit) { + Ok(_) => {}, + Err(e) => println!("Set file rlimit to {} failed {}.", fd_limit, e), + } +} + +#[tokio::main] +async fn main() { + let args: Vec = env::args().collect(); + if args.len() != 3 { + let bin: String = args[0].trim().parse().expect("Binary name error"); + println!("Usage example:"); + println!(" {} 192.168.1.10:12310 10", bin); + return; + } + set_rlimit_fd().await; + let addr: String = args[1].trim().parse().expect("Input address: '192.168.1.10:12310'"); + let max_block_threads: usize = args[2].trim().parse().expect("Input max blocking threads number in arg 2: like '10'"); + let listener = TcpListener::bind(addr.clone()).unwrap(); + let async_listener = AsyncTcpListener::from_std(listener).unwrap(); + let runtime = Builder::new_multi_thread() + .max_blocking_threads(max_block_threads) + .enable_all() + .build() + .unwrap(); + + println!("Ready to listen addr:{}, max blocking threads:{}", addr, max_block_threads); + + let mut coroutine_idx: usize = 1; + loop { + let (s, _) = async_listener.accept().await.unwrap(); + let cur_idx = coroutine_idx.clone(); + coroutine_idx += 1; + match Some(s) { + Some(stream) => { + // 收到一个新的fifo连接请求，拉起新的协程处理函数 + runtime.spawn(cofifo::qtfs_fifo_server(stream, cur_idx)); + } + _ => { + eprintln!("Accept error!"); + } + } + } +} \ No newline at end of file diff --git a/qtfs/qtinfo/Makefile b/qtfs/qtinfo/Makefile new file mode 100644 index 0000000..5e75e6e --- /dev/null +++ b/qtfs/qtinfo/Makefile @@ -0,0 +1,23 @@ +CFLAGS += -g -O2 +CFLAGS += -fstack-protector-strong +CFLAGS += -fPIE -pie -fPIC +CFLAGS += -D_FORTIFY_SOURCE=2 +LDFLAGS += -Wl,-z,now +LDFLAGS += -Wl,-z,noexecstack +LDFLAGS += -fPIE -pie + +all: qtinfo qtcfg + +qtinfo: + gcc $(CFLAGS) $(LDFLAGS) -D$(role) -o qtinfo qtinfo.c -I../ -I../include/ + +qtcfg: + gcc $(CFLAGS) $(LDFLAGS) -DQTINFO_RELEASE -D$(role) -o qtcfg qtinfo.c -I../ -I../include/ + +install: + yes | cp qtinfo /usr/bin/ + yes | cp qtcfg /usr/bin/ + +clean: + rm -rf qtcfg qtinfo qtinfo.o + diff --git a/qtfs/qtinfo/qtinfo.c b/qtfs/qtinfo/qtinfo.c new file mode 100644 index 0000000..4c915ae --- /dev/null +++ b/qtfs/qtinfo/qtinfo.c @@ -0,0 +1,627 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * qtfs licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Liqiang + * Create: 2023-03-20 + * Description: + *******************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "qtinfo.h" +#include "comm.h" +#include "ipc/uds_module.h" + +#ifdef client +#define QTFS_DEV_NAME "/dev/qtfs_client" +#else +#define QTFS_DEV_NAME "/dev/qtfs_server" +#endif + +#define qtinfo_out(info, ...) \ + do {\ + printf(info"\n", ##__VA_ARGS__);\ + } while (0); + +#define qtinfo_out2(info, ...) \ + do {\ + printf(info, ##__VA_ARGS__);\ + } while (0); + +#define qtinfo_err(info, ...) \ + do {\ + printf("ERROR: "info"\n", ##__VA_ARGS__);\ + } while (0); + +#define RECV_BUFF_LEN 256 + +struct qtinfo_type_str qtinfo_all_events[] = { + {QTFS_REQ_NULL, "null"}, + {QTFS_REQ_MOUNT, "mount"}, + {QTFS_REQ_OPEN, "open"}, + {QTFS_REQ_CLOSE, "close"}, + {QTFS_REQ_READ, "read"}, + {QTFS_REQ_READITER, "readiter"}, //5 + {QTFS_REQ_WRITE, "write"}, + {QTFS_REQ_LOOKUP, "lookup"}, + {QTFS_REQ_READDIR, "readdir"}, + {QTFS_REQ_MKDIR, "mkdir"}, + {QTFS_REQ_RMDIR, "rmdir"}, //10 + {QTFS_REQ_GETATTR, "getattr"}, + {QTFS_REQ_SETATTR, "setattr"}, + {QTFS_REQ_ICREATE, "icreate"}, + {QTFS_REQ_MKNOD, "mknod"}, + {QTFS_REQ_UNLINK, "unlink"}, //15 + {QTFS_REQ_SYMLINK, "symlink"}, + {QTFS_REQ_LINK, "link"}, + {QTFS_REQ_GETLINK, "getlink"}, + {QTFS_REQ_READLINK, "readlink"}, + {QTFS_REQ_RENAME, "rename"}, //20 + + {QTFS_REQ_XATTRLIST, "xattrlist"}, + {QTFS_REQ_XATTRGET, "xattrget"}, + {QTFS_REQ_XATTRSET, "xattrset"}, + + {QTFS_REQ_SYSMOUNT, "sysmount"}, + {QTFS_REQ_SYSUMOUNT, "sysumount"}, //25 + {QTFS_REQ_FIFOPOLL, "fifo_poll"}, + + {QTFS_REQ_STATFS, "statfs"}, + {QTFS_REQ_IOCTL, "ioctl"}, + {QTFS_REQ_EPOLL_CTL, "epollctl"}, + + {QTFS_REQ_EPOLL_EVENT, "epollevent"}, // 30 + {QTFS_REQ_LLSEEK, "llseek"}, + {QTFS_SC_KILL, "sc_kill"}, + {QTFS_SC_SCHED_GETAFFINITY, "sc_getaffi"}, + {QTFS_SC_SCHED_SETAFFINITY, "sc_setaffi"}, +}; + +static void qtinfo_events_count(struct qtinfo *evts) +{ + unsigned long total = 0; + int i; +#ifdef client + qtinfo_out("++++++++++++++++++++++++++send events count++++++++++++++++++++++++++"); + for (i = 0; i < (sizeof(qtinfo_all_events) / sizeof(struct qtinfo_type_str)) - 3; i += 3) { + qtinfo_out("%-10s: %-10lu %-10s: %-10lu %-10s: %-10lu", + qtinfo_all_events[i].str, evts->c.o_events[i], + qtinfo_all_events[i+1].str, evts->c.o_events[i+1], + qtinfo_all_events[i+2].str, evts->c.o_events[i+2]); + total += evts->c.o_events[i] + evts->c.o_events[i+1] + evts->c.o_events[i+2]; + } + for (; i < sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str); i++) { + qtinfo_out2("%-10s: %-10lu ", qtinfo_all_events[i].str, evts->c.o_events[i]); + total += evts->c.o_events[i]; + } + qtinfo_out2("\n"); + qtinfo_out("Send events total: %lu", total); + total = 0; + + qtinfo_out("++++++++++++++++++++++++++recv events count++++++++++++++++++++++++++"); + for (i = 0; i < (sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str)) - 3; i+=3) { + qtinfo_out("%-10s: %-10lu %-10s: %-10lu %-10s: %-10lu", + qtinfo_all_events[i].str, evts->c.i_events[i], + qtinfo_all_events[i+1].str, evts->c.i_events[i+1], + qtinfo_all_events[i+2].str, evts->c.i_events[i+2]); + total += evts->c.i_events[i] + evts->c.i_events[i+1] + evts->c.i_events[i+2]; + } + for(; i < sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str); i++) { + qtinfo_out2("%-10s: %-10lu ", qtinfo_all_events[i].str, evts->c.i_events[i]); + total += evts->c.i_events[i]; + } + qtinfo_out2("\n"); + qtinfo_out("Recv events total: %lu", total); + total = 0; + qtinfo_out("++++++++++++++++++++++++++send error count+++++++++++++++++++++++++++"); + for(i = 0; i < (sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str)) - 3; i+=3) { + qtinfo_out("%-10s: %-10lu %-10s: %-10lu %-10s: %-10lu", + qtinfo_all_events[i].str, evts->c.send_err[i], + qtinfo_all_events[i+1].str, evts->c.send_err[i+1], + qtinfo_all_events[i+2].str, evts->c.send_err[i+2]); + total += evts->c.send_err[i] + evts->c.send_err[i+1] + evts->c.send_err[i+2]; + } + for(; i < sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str); i++) { + qtinfo_out2("%-10s: %-10lu ", qtinfo_all_events[i].str, evts->c.send_err[i]); + total += evts->c.send_err[i]; + } + qtinfo_out2("\n"); + qtinfo_out("Send events error total: %lu", total); + total = 0; + + qtinfo_out("++++++++++++++++++++++++++recv error count+++++++++++++++++++++++++++"); + for(i = 0; i < (sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str)) - 3; i+=3) { + qtinfo_out("%-10s: %-10lu %-10s: %-10lu %-10s: %-10lu", + qtinfo_all_events[i].str, evts->c.recv_err[i], + qtinfo_all_events[i+1].str, evts->c.recv_err[i+1], + qtinfo_all_events[i+2].str, evts->c.recv_err[i+2]); + total += evts->c.recv_err[i] + evts->c.recv_err[i+1] + evts->c.recv_err[i+2]; + } + for(; i < sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str); i++) { + qtinfo_out2("%-10s: %-10lu ", qtinfo_all_events[i].str, evts->c.recv_err[i]); + total += evts->c.recv_err[i]; + } + qtinfo_out2("\n"); + qtinfo_out("Recv events error total: %lu", total); + total = 0; +#endif +#ifdef server + qtinfo_out("++++++++++++++++++++++++++send events count++++++++++++++++++++++++++"); + for (i = 0; i < (sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str)) - 3; i+=3) { + qtinfo_out("%-10s: %-10lu %-10s: %-10lu %-10s: %-10lu", + qtinfo_all_events[i].str, evts->s.o_events[i], + qtinfo_all_events[i+1].str, evts->s.o_events[i+1], + qtinfo_all_events[i+2].str, evts->s.o_events[i+2]); + total += evts->s.o_events[i] + evts->s.o_events[i+1] + evts->s.o_events[i+2]; + } + for (; i < sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str); i++) { + qtinfo_out2("%-10s: %-10lu ", qtinfo_all_events[i].str, evts->s.o_events[i]); + total += evts->s.o_events[i]; + } + qtinfo_out2("\n"); + qtinfo_out("Send events total: %lu", total); + total = 0; + + qtinfo_out("++++++++++++++++++++++++++recv events count++++++++++++++++++++++++++"); + for (i = 0; i < (sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str)) - 3; i+=3) { + qtinfo_out("%-10s: %-10lu %-10s: %-10lu %-10s: %-10lu", + qtinfo_all_events[i].str, evts->s.i_events[i], + qtinfo_all_events[i+1].str, evts->s.i_events[i+1], + qtinfo_all_events[i+2].str, evts->s.i_events[i+2]); + total += evts->s.i_events[i] + evts->s.i_events[i+1] + evts->s.i_events[i+2]; + } + for(; i < sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str); i++) { + qtinfo_out2("%-10s: %-10lu ", qtinfo_all_events[i].str, evts->s.i_events[i]); + total += evts->s.i_events[i]; + } + qtinfo_out2("\n"); + qtinfo_out("Recv events total: %lu", total); + + qtinfo_out("++++++++++++++++++++++++++req check err+++++++++++++++++++++++++++++++"); + for (i = 0; i < (sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str)) - 3; i+=3) { + qtinfo_out("%-10s: %-10lu %-10s: %-10lu %-10s: %-10lu", + qtinfo_all_events[i].str, evts->s.req_check[i], + qtinfo_all_events[i+1].str, evts->s.req_check[i+1], + qtinfo_all_events[i+2].str, evts->s.req_check[i+2]); + } + for(; i < sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str); i++) { + qtinfo_out2("%-10s: %-10lu ", qtinfo_all_events[i].str, evts->s.req_check[i]); + } + qtinfo_out("\n+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++") +#endif +} + +static void qtinfo_misc_count(struct qtinfo *info) +{ + qtinfo_out("++++++++++++++++++++++++++++++Misc count+++++++++++++++++++++++++++++"); +#ifdef client + qtinfo_out("Active connects: %-8lu Seq err count : %-8lu Restartsys : %-8lu", + info->c.cnts[QTINF_ACTIV_CONN], info->c.cnts[QTINF_SEQ_ERR], info->c.cnts[QTINF_RESTART_SYS]); + qtinfo_out("Type mismatch : %-8lu Epoll add fds : %-8lu Epoll del fds: %-8lu", + info->c.cnts[QTINF_TYPE_MISMATCH], info->c.cnts[QTINF_EPOLL_ADDFDS], info->c.cnts[QTINF_EPOLL_DELFDS]); + qtinfo_out("Epoll err fds : %-8lu", + info->c.cnts[QTINF_EPOLL_FDERR]); +#else + qtinfo_out("Active connects: %-8lu Epoll add fds: %-8lu Epoll del fds: %-8lu", + info->s.cnts[QTINF_ACTIV_CONN], info->s.cnts[QTINF_EPOLL_ADDFDS], info->s.cnts[QTINF_EPOLL_DELFDS]); +#endif +} + +static void qtinfo_thread_state(struct qtinfo *info) +{ + int i = 0; + + qtinfo_out("+++++++++++++++++++++++++++Connection state++++++++++++++++++++++++++"); + for (i = 0; i < QTFS_MAX_THREADS - 3; i+=3) { + qtinfo_out("Conn%-2d: %-10s Conn%-2d: %-10s Conn%-2d: %-10s", + i+1, QTINFO_STATE(info->thread_state[i]), + i+2, QTINFO_STATE(info->thread_state[i+1]), + i+3, QTINFO_STATE(info->thread_state[i+2])); + } + for (; i < QTFS_MAX_THREADS; i++) { + qtinfo_out2("Conn%-2d: %-10s ", i+1, QTINFO_STATE(info->thread_state[i])); + } + qtinfo_out("Epoll state: %-10s", QTINFO_STATE(info->epoll_state)); + return; +} + +static void qtinfo_pvar_count(struct qtinfo *info) +{ + int i = 0; + qtinfo_out("+++++++++++++++++++++++++++++Param count+++++++++++++++++++++++++++++"); + qtinfo_out("Parameter valid count: %-2d Parameter busy count: %-2d", + info->pvar_vld, info->pvar_busy); + for (i = 0; i < QTFS_MAX_THREADS; i++) { + qtinfo_out("Conn%-2d holder: [%-20s]", i+1, (info->who_using[i][0] == '\0') ? "No one" : info->who_using[i]); + } +} + +static void qtinfo_log_level(struct qtinfo *info) +{ + qtinfo_out("Log level: %d", info->log_level); +} + +static int qtinfo_opt_a(int fd) +{ + struct qtinfo *diag = (struct qtinfo *)malloc(sizeof(struct qtinfo)); + if (diag == NULL) { + qtinfo_err("malloc failed."); + return -1; + } + memset(diag, 0, sizeof(struct qtinfo)); + int ret = ioctl(fd, QTFS_IOCTL_ALLINFO, diag); + if (ret != QTOK) { + qtinfo_err("ioctl failed, ret:%d.", ret); + free(diag); + return -1; + } + qtinfo_events_count(diag); + qtinfo_misc_count(diag); + qtinfo_log_level(diag); + qtinfo_thread_state(diag); + qtinfo_pvar_count(diag); + free(diag); + return 0; +} + +static int qtinfo_opt_c(int fd) +{ + int ret = ioctl(fd, QTFS_IOCTL_CLEARALL, NULL); + return ret; +} + +static int qtinfo_opt_l(int fd, char *level) +{ + int ret; + + ret = ioctl(fd, QTFS_IOCTL_LOGLEVEL, level); + if (ret != QTOK) { + qtinfo_err("Set qtfs log level:%s failed.", level); + return ret; + } + qtinfo_out("Set qtfs log level to %s success.", level); + return ret; +} + +static int qtinfo_opt_t(int fd) +{ + int i; + struct qtinfo *diag = (struct qtinfo *)malloc(sizeof(struct qtinfo)); + if (diag == NULL) { + qtinfo_err("malloc failed."); + return -1; + } + int ret = ioctl(fd, QTFS_IOCTL_ALLINFO, (unsigned long)diag); + qtinfo_out("++++++++++++++++++++++++++qtreq_xxx size++++++++++++++++++++++++++"); + for (i = 0; i < (sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str)) - 3; i+=3) { + qtinfo_out("%-10s req: %-10lu %-10s req: %-10lu %-10s req: %-10lu", + qtinfo_all_events[i].str, diag->req_size[i], + qtinfo_all_events[i+1].str, diag->req_size[i+1], + qtinfo_all_events[i+2].str, diag->req_size[i+2]); + } + for (; i < sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str); i++) { + qtinfo_out2("%-10s req: %-10lu ", qtinfo_all_events[i].str, diag->req_size[i]); + } + qtinfo_out2("\n"); + qtinfo_out("++++++++++++++++++++++++++qtrsp_xxx size++++++++++++++++++++++++++"); + for (i = 0; i < (sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str)) - 3; i+=3) { + qtinfo_out("%-10s rsp: %-10lu %-10s rsp: %-10lu %-10s rsp: %-10lu", + qtinfo_all_events[i].str, diag->rsp_size[i], + qtinfo_all_events[i+1].str, diag->rsp_size[i+1], + qtinfo_all_events[i+2].str, diag->rsp_size[i+2]); + } + for (; i < sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str); i++) { + qtinfo_out2("%-10s rsp: %-10lu ", qtinfo_all_events[i].str, diag->rsp_size[i]); + } + qtinfo_out2("\n"); + + free(diag); + return ret; +} + +static int qtinfo_opt_p(int fd, char *support) +{ + int ret; + int sup = atoi(support); + + ret = ioctl(fd, QTFS_IOCTL_EPOLL_SUPPORT, sup); + if (ret != QTOK) { + qtinfo_out("Set qtfs epoll support to:%s failed.", (sup == 1) ? "any file" : "fifo file"); + return ret; + } + qtinfo_out("Set qtfs epoll support to %s success.", (sup == 1) ? "any file" : "fifo file"); + return ret; +} + +#define PATH_MAX 4096 +static int qtinfo_opt_u() +{ + int ret = -1; + int len; + struct sockaddr_un svr; + char buf[RECV_BUFF_LEN]; + int sockfd = socket(AF_UNIX, SOCK_STREAM, 0); + if (sockfd < 0) { + qtinfo_err("Create socket fd failed."); + return -1; + } + + memset(&svr, 0, sizeof(svr)); + svr.sun_family = AF_UNIX; + strcpy(svr.sun_path, UDS_DIAG_ADDR); + len = offsetof(struct sockaddr_un, sun_path) + strlen(svr.sun_path); + if (connect(sockfd, (struct sockaddr *)&svr, len) < 0) { + qtinfo_err("connect to %s failed.", UDS_DIAG_ADDR); + close(sockfd); + return -1; + } + while (1) { + memset(buf, 0, RECV_BUFF_LEN); + ret = recv(sockfd, buf, RECV_BUFF_LEN, 0); + if (ret <= 0) + break; + qtinfo_out2("%s", buf); + } + qtinfo_out2("\n"); + close(sockfd); + return ret; +} + +static int qtinfo_opt_s() +{ + int ret = -1; + int len; + struct sockaddr_un svr; + char buf[RECV_BUFF_LEN]; + int sockfd = socket(AF_UNIX, SOCK_STREAM, 0); + if (sockfd < 0) { + qtinfo_err("Create socket fd failed."); + return -1; + } + + memset(&svr, 0, sizeof(svr)); + svr.sun_family = AF_UNIX; + strcpy(svr.sun_path, UDS_LOGLEVEL_UPD); + len = offsetof(struct sockaddr_un, sun_path) + strlen(svr.sun_path); + if (connect(sockfd, (struct sockaddr *)&svr, len) < 0) { + qtinfo_err("connect to %s failed.", UDS_LOGLEVEL_UPD); + close(sockfd); + return -1; + } + while (1) { + memset(buf, 0, RECV_BUFF_LEN); + ret = recv(sockfd, buf, RECV_BUFF_LEN, 0); + if (ret <= 0) + break; + qtinfo_out2("%s", buf); + } + qtinfo_out2("\n"); + close(sockfd); + return ret; + +} + +char wl_type_str[QTFS_WHITELIST_MAX][16] = { +#ifdef server + "Open", + "Write", + "Read", + "Readdir", + "Mkdir", + "Rmdir", + "Create", + "Unlink", + "Rename", + "Setattr", + "Setxattr", + "Mount", + "Kill", +#endif + "Udsconnect" +}; + +int qtinfo_match_cap(char *cap) +{ + if (cap == NULL) + return -1; + for (int type = 0; type < QTFS_WHITELIST_MAX; type++) { + if (strcasecmp(wl_type_str[type], cap) == 0) + return type; + } + return -1; +} + +#define PATH_MAX 4096 +static int qtinfo_opt_x(int fd, char *path, char *cap) +{ + int ret = -1; + int index = 0; + struct qtfs_wl_item head; + ret = qtinfo_match_cap(cap); + if (ret < 0) { + qtinfo_err("White list add type:%s unknown.", cap); + return -1; + } + head.type = ret; + head.len = strlen(path); + if (head.len >= PATH_MAX || head.len == 0) { + qtinfo_err("White list add len:%u invalid", head.len); + return -1; + } + head.path = path; + ret = ioctl(fd, QTFS_IOCTL_WL_ADD, &head); + if (ret != QTOK) { + qtinfo_err("ioctl add white list failed"); + ret = -1; + } else { + qtinfo_out("successed to add white list item:%s successed", path); + ret = 0; + } + return ret; +} + +static int qtinfo_opt_y(int fd, char *index, char *cap) +{ + struct qtfs_wl_item head; + int ret = qtinfo_match_cap(cap); + if (ret < 0) { + qtinfo_err("White list delete type:%s unknown.", cap); + return -1; + } + head.index = atoi(index); + head.type = ret; + head.path = NULL; + head.len = 0; + ret = ioctl(fd, QTFS_IOCTL_WL_DEL, &head); + if (ret != QTOK) { + qtinfo_err("failed to delete white list index:%d", head.index); + return -1; + } else { + qtinfo_out("successed to delete white list index:%d", head.index); + } + return 0; +} + +static void qtinfo_opt_z_bytype(int fd, unsigned int type) +{ + int ret; + char query[PATH_MAX]; + struct qtfs_wl_item head; + head.path = query; + head.type = type; + qtinfo_out("Get qtfs <%s> white list:", wl_type_str[type]); + for (unsigned int index = 0; index < QTFS_WL_MAX_NUM; index++) { + memset(head.path, 0, PATH_MAX); + head.index = index; + ret = ioctl(fd, QTFS_IOCTL_WL_GET, &head); + if (ret != QTOK) + break; + qtinfo_out(" [index:%u][path:%s]", index, head.path); + } + return; +} + +static int qtinfo_opt_z(int fd, char *cap) +{ + int ret = -1; + int index = 0; + ret = qtinfo_match_cap(cap); + + if (ret >= 0 && ret < QTFS_WHITELIST_MAX) { + qtinfo_opt_z_bytype(fd, ret); + return 0; + } + for (int i = 0; i < QTFS_WHITELIST_MAX; i++) { + qtinfo_opt_z_bytype(fd, i); + } + return 0; +} + +static void qtinfo_help(char *exec) +{ + qtinfo_out("Usage: %s [OPTION].", exec); + qtinfo_out("Display qtfs client/server diagnostic information."); +#ifndef QTINFO_RELEASE + qtinfo_out(" -a, All diag info."); + qtinfo_out(" -c, Clear all diag info."); + qtinfo_out(" -l, Set log level(valid param: \"NONE\", \"ERROR\", \"WARN\", \"INFO\", \"DEBUG\")."); + qtinfo_out(" -t, For test informations."); + qtinfo_out(" -p, Epoll support file mode(1: any files; 0: only fifo)."); + qtinfo_out(" -u, Display unix socket proxy diagnostic info"); + qtinfo_out(" -s, Set unix socket proxy log level(Increase by 1 each time)"); +#endif +#ifdef server + qtinfo_out(" -w, White list type(open/write/read/readdir/mkdir/rmdir/create/\n" + " unlink/rename/setattr/setxattr/mount/kill/udsconnect)"); +#endif +#ifdef client + qtinfo_out(" -w, White list type(udsconnect)"); +#endif + qtinfo_out(" -x, Add a qtfs white list path(example: -x /home/, must use with -w)"); + qtinfo_out(" -y, Delete a qtfs white list with index(example: -y 1, must use with -w)"); + qtinfo_out(" -z, Get all qtfs white list(just use -z, or with white list type)"); +} + +int main(int argc, char *argv[]) +{ +#define MAX_CAP_LEN 16 + int ret = -1; + int ch; + char wl_cap[MAX_CAP_LEN] = {0}; + if ((argc == 1) || (argc == 2 && strcmp(argv[1], "--help") == 0)) { + qtinfo_help(argv[0]); + return -1; + } + int fd = open(QTFS_DEV_NAME, O_RDONLY|O_NONBLOCK); + if (fd < 0) { + qtinfo_err("open file %s failed.", QTFS_DEV_NAME); +#ifdef QTINFO_RELEASE + return -1; +#endif + } +#ifndef QTINFO_RELEASE + while ((ch = getopt(argc, argv, "acl:tp:usw:x:y:z::")) != -1) { +#else + while ((ch = getopt(argc, argv, "w:x:y:z::")) != -1) { +#endif + switch (ch) { +#ifndef QTINFO_RELEASE + case 'a': + ret = qtinfo_opt_a(fd); + break; + case 'c': + ret = qtinfo_opt_c(fd); + break; + case 'l': + ret = qtinfo_opt_l(fd, optarg); + break; + case 't': + ret = qtinfo_opt_t(fd); + break; + case 'p': + ret = qtinfo_opt_p(fd, optarg); + break; + case 'u': + ret = qtinfo_opt_u(); + break; + case 's': + ret = qtinfo_opt_s(); + break; +#endif + case 'w': + strncpy(wl_cap, optarg, MAX_CAP_LEN - 1); + break; + case 'x': + ret = qtinfo_opt_x(fd, optarg, wl_cap); + break; + case 'y': + ret = qtinfo_opt_y(fd, optarg, wl_cap); + break; + case 'z': + ret = qtinfo_opt_z(fd, optarg); + break; + default: + ret = 0; + qtinfo_help(argv[0]); + break; + } + } + + close(fd); + return ret; +} diff --git a/qtfs/qtinfo/qtinfo.h b/qtfs/qtinfo/qtinfo.h new file mode 100644 index 0000000..533133b --- /dev/null +++ b/qtfs/qtinfo/qtinfo.h @@ -0,0 +1,78 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * qtfs licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Liqiang + * Create: 2023-03-20 + * Description: + *******************************************************************************/ + +#ifndef __QTINFO_H__ +#define __QTINFO_H__ + + +enum qtfs_req_type +{ + QTFS_REQ_NULL, + QTFS_REQ_MOUNT, + QTFS_REQ_OPEN, + QTFS_REQ_CLOSE, + QTFS_REQ_READ, + QTFS_REQ_READITER, // 5 + QTFS_REQ_WRITE, + QTFS_REQ_LOOKUP, + QTFS_REQ_READDIR, + QTFS_REQ_MKDIR, + QTFS_REQ_RMDIR, // 10 + QTFS_REQ_GETATTR, + QTFS_REQ_SETATTR, + QTFS_REQ_ICREATE, + QTFS_REQ_MKNOD, + QTFS_REQ_UNLINK, // 15 + QTFS_REQ_SYMLINK, + QTFS_REQ_LINK, + QTFS_REQ_GETLINK, + QTFS_REQ_READLINK, + QTFS_REQ_RENAME, // 20 + + QTFS_REQ_XATTRLIST, + QTFS_REQ_XATTRGET, + QTFS_REQ_XATTRSET, + + QTFS_REQ_SYSMOUNT, + QTFS_REQ_SYSUMOUNT, // 25 + QTFS_REQ_FIFOPOLL, + + QTFS_REQ_STATFS, + QTFS_REQ_IOCTL, + + QTFS_REQ_EPOLL_CTL, + + QTFS_REQ_EPOLL_EVENT, + + QTFS_REQ_LLSEEK, + + // REMOTE SYSCALL + QTFS_SC_KILL, + QTFS_SC_SCHED_GETAFFINITY, + QTFS_SC_SCHED_SETAFFINITY, + + QTFS_REQ_EXIT, + QTFS_REQ_INV, +}; + +#define MAX_QTINFO_TYPE_STR_LEN 20 + +struct qtinfo_type_str { + enum qtfs_req_type type; + char str[MAX_QTINFO_TYPE_STR_LEN]; +}; + +#endif + diff --git a/qtfs/rexec/Makefile b/qtfs/rexec/Makefile new file mode 100644 index 0000000..02e8e02 --- /dev/null +++ b/qtfs/rexec/Makefile @@ -0,0 +1,26 @@ +DEPGLIB=-lglib-2.0 -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include + +CFLAGS += -g -O2 +CFLAGS += -fstack-protector-strong +CFLAGS += -fPIE -pie -fPIC +CFLAGS += -D_FORTIFY_SOURCE=2 +LDFLAGS += -Wl,-z,now +LDFLAGS += -Wl,-z,noexecstack +LDFLAGS += -fPIE -pie + +all: rexec rexec_server + +rexec : + gcc $(CFLAGS) $(LDFLAGS) -o rexec rexec.c rexec_sock.c -ljson-c + +rexec_server : + gcc $(CFLAGS) $(LDFLAGS) -o rexec_server rexec_server.c rexec_sock.c rexec_shim.c -ljson-c $(DEPGLIB) +test: + go test -v ./common_test.go ./common.go + +install: + yes | cp -f rexec /usr/bin/ + yes | cp -f rexec_server /usr/bin/ + +clean: + rm -rf rexec rexec_server diff --git a/qtfs/rexec/README.md b/qtfs/rexec/README.md new file mode 100644 index 0000000..d25eda3 --- /dev/null +++ b/qtfs/rexec/README.md @@ -0,0 +1,36 @@ +# rexec + +rexec is a sample application demonstrating libchan nested channels and +bytestreams through remote executation calls. This is a minimal +implementation to demonstrate the usage of libchan. + +### Usage + +Server +~~~~ +$ cd rexec_server +$ go build . +$ ./rexec_server +~~~~ + +Client +~~~~ +$ go build . +$ ./rexec /bin/echo "hello" +hello +$ ./rexec /bin/sh -c "exit 4" +$ echo $? +4 +~~~~ + +### Usage with TLS +Server +~~~~ +$ TLS_CERT=./cert.pem TLS_KEY=./key.pem ./rexec_server +~~~~ + +Client +~~~~ +$ USE_TLS=true ./rexec /bin/echo "hello" +hello +~~~~ diff --git a/qtfs/rexec/rexec.c b/qtfs/rexec/rexec.c new file mode 100644 index 0000000..060abc5 --- /dev/null +++ b/qtfs/rexec/rexec.c @@ -0,0 +1,773 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * qtfs licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Liqiang + * Create: 2023-03-20 + * Description: + *******************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dirent.h" + +#include "rexec_sock.h" +#include "rexec.h" + +#define REXEC_MSG_LEN 1024 +FILE *rexec_logfile = NULL; + +struct rexec_global_var { + int rexec_hs_fd[2]; +}; + +struct rexec_thread_arg { + int efd; + int connfd; + char **argv; +}; + +struct rexec_global_var g_rexec; + + +struct rexec_client_event { + int fd; + int outfd; // for stdin out err and other pipe + int (*handler)(struct rexec_client_event *); + int *exit_status; + int *pidfd; +}; + +#define REXEC_PIDMAP_PATH "/var/run/rexec/pids" +#define REXEC_PIDMAP_PATH_LEN 64 +#define REXEC_PID_LEN 16 +static int rexec_conn_to_server() +{ + struct rexec_conn_arg arg; + char *ret = strncpy(arg.sun_path, REXEC_UDS_CONN, sizeof(arg.sun_path)); + if (ret == NULL) { + rexec_err("strncpy sun path failed"); + return -1; + } + arg.cs = REXEC_SOCK_CLIENT; + arg.udstype = SOCK_STREAM; + if (rexec_build_unix_connection(&arg) != 0) + return -1; + return arg.connfd; +} + +static int rexec_calc_argv_len(int argc, char *argv[]) +{ + int len = 0; + for (int i = 0; i < argc; i++) { + if (argv[i] == NULL) { + rexec_err("Invalid argv index:%d", i); + return -1; + } + len += strlen(argv[i]); + len++; + } + return len; +} + +static int rexec_msg_fill_argv(int argc, char *argv[], char *msg) +{ + int offset = 0; + for (int i = 0; i < argc; i++) { + strcpy(&msg[offset], argv[i]); //此处msg已经在前面通过计算出的len预先分配内存，保证这里不会越界 + offset += (strlen(argv[i]) + 1); + } + return offset; +} + +static int rexec_io(struct rexec_client_event *evt) +{ +#define MAX_MSG_LEN 256 + char buf[MAX_MSG_LEN]; + int len; + int ret; + while ((len = read(evt->fd, buf, MAX_MSG_LEN)) > 0) { + ret = write(evt->outfd, buf, len); + if (ret <= 0) { + rexec_err("Read from fd:%d len:%d write to fd:%d failed ret:%d", evt->fd, len, evt->outfd, ret); + return REXEC_EVENT_DEL; + } + if (ret != len) { + rexec_err("Read from fd:%d len:%d but write to fd:%d ret:%d", evt->fd, len, evt->outfd, ret); + } + } + return REXEC_EVENT_OK; +} + +// return -1 means process exit. +static int rexec_conn_msg(struct rexec_client_event *evt) +{ + struct rexec_msg head; + int ret = recv(evt->fd, &head, sizeof(struct rexec_msg), MSG_WAITALL); + if (ret <= 0) { + rexec_err("Rexec conn recv err:%d errno:%d", ret, errno); + return REXEC_EVENT_EXIT; + } + switch (head.msgtype) { + case REXEC_KILL: + *evt->exit_status = head.exit_status; + rexec_err("Rexec conn recv kill msg, exit:%d now.", head.exit_status); + return REXEC_EVENT_EXIT; + case REXEC_PIDMAP: { + int mypid = getpid(); + int peerpid = head.pid; + char path[REXEC_PIDMAP_PATH_LEN] = {0}; + char buf[REXEC_PID_LEN] = {0}; + int fd; + int err; + if (*evt->pidfd > 0) { + rexec_err("Rexec pidmap msg > 1 error."); + return REXEC_EVENT_OK; + } + sprintf(path, "%s/%d", REXEC_PIDMAP_PATH, mypid); + fd = open(path, O_CREAT|O_WRONLY, 0600); + if (fd < 0) { + rexec_err("Rexec create pidmap:%d-%d failed, path:%s open failed:%d", + mypid, peerpid, path, fd); + break; + } + *evt->pidfd = fd; + if ((err = flock(fd, LOCK_EX)) != 0) { + rexec_err("Rexec flock file:%s failed, errno:%d rexec exit.", path, err); + return REXEC_EVENT_EXIT; + } + if ((err = ftruncate(fd, 0)) != 0) { + rexec_err("Rexec pidmap file:%s clear failed errno:%d rexec exit.", path, err); + return REXEC_EVENT_EXIT; + } + if ((err = lseek(fd, 0, SEEK_SET)) < 0) { + rexec_err("Rexec pidmap file:%s lseek 0 failed errno:%d rexec exit", path, err); + return REXEC_EVENT_EXIT; + } + sprintf(buf, "%d", peerpid); + if ((err = write(fd, buf, strlen(buf))) <= 0) { + rexec_err("Rexec pidmap file:%s write pid:%d failed errno:%d rexec exit.", path, peerpid, err); + return REXEC_EVENT_EXIT; + } + if (g_rexec.rexec_hs_fd[PIPE_WRITE] != -1 && g_rexec.rexec_hs_fd[PIPE_READ] != -1) { + err = write(g_rexec.rexec_hs_fd[PIPE_WRITE], "1", 1); + if (err <= 0) { + rexec_err("rexec handshake write 1 failed, hs write:%d.", g_rexec.rexec_hs_fd[PIPE_WRITE]); + return REXEC_EVENT_ERR; + } + } else { + char msg[sizeof(struct rexec_msg) + 1]; + struct rexec_msg *hs = msg; + char *ok = hs->msg; + hs->msgtype = REXEC_HANDSHAKE; + hs->msglen = 1; + *ok = '1'; + if (write(evt->fd, hs, sizeof(struct rexec_msg) + 1) <= 0) { + rexec_err("send handshake failed, remote process will die"); + return REXEC_EVENT_EXIT; + } + } + break; + } + default: + break; + } + + rexec_log("Rexec conn recv msgtype:%d argc:%d pipefd:%d msglen:%d", + head.msgtype, head.argc, head.pipefd, head.msglen); + return REXEC_EVENT_OK; +} + +static struct rexec_client_event *rexec_add_event(int efd, int fd, int outfd, int (*handler)(struct rexec_client_event *)) +{ + struct rexec_client_event *event = (struct rexec_client_event *)malloc(sizeof(struct rexec_client_event)); + if (event == NULL) { + rexec_err("malloc failed."); + return NULL; + } + event->fd = fd; + event->outfd = outfd; + event->handler = handler; + struct epoll_event evt; + evt.data.ptr = (void *)event; + evt.events = EPOLLIN; + if (-1 == epoll_ctl(efd, EPOLL_CTL_ADD, event->fd, &evt)) { + rexec_err("epoll ctl add fd:%d event failed.", event->fd); + free(event); + return NULL; + } + return event; +} + +static int rexec_del_event(struct rexec_client_event *event) +{ + // close will del fd in epoll list + close(event->fd); + free(event); + return 0; +} + +enum { + REPOL_IN_INDEX = 0, + REPOL_OUT_INDEX, + REPOL_ERR_INDEX, + REPOL_INV_INDEX, +}; +static int rexec_std_event(int efd, int rstdin, int rstdout, int rstderr) +{ + #define REXEC_MAX_EVENTS 4 + int infds[REPOL_INV_INDEX] = {STDIN_FILENO, rstdout, rstderr}; + int outfds[REPOL_INV_INDEX] = {rstdin, STDOUT_FILENO, STDERR_FILENO}; + + for (int i = 0; i < REPOL_INV_INDEX; i++) { + if (NULL == rexec_add_event(efd, infds[i], outfds[i], rexec_io)) { + rexec_err("epoll ctl add fd:%d event failed and ignore this mistake.", infds[i]); + continue; + } else { + if (rexec_set_nonblock(infds[i], 1) != 0) { + rexec_err("rexec set fd:%d i:%d non block failed.", infds[i], i); + return -1; + } + } + } + return 0; +} + +static void rexec_event_run(int efd) +{ + struct epoll_event *evts = calloc(REXEC_MAX_EVENTS, sizeof(struct epoll_event)); + if (evts == NULL) { + rexec_err("init calloc evts failed."); + return; + } + while (1) { + int n = epoll_wait(efd, evts, REXEC_MAX_EVENTS, 1000); + int process_exit = 0; + if (n == 0) + continue; + if (n < 0) { + rexec_err("epoll wait return errcode:%d", n); + continue; + } + for (int i = 0; i < n; i++) { + struct rexec_client_event *evt = (struct rexec_client_event *)evts[i].data.ptr; + int ret = evt->handler(evt); + if (ret == REXEC_EVENT_EXIT) { + process_exit = 1; + } + if (ret == REXEC_EVENT_DEL) { + rexec_del_event(evt); + } + } + // process will exit, and free all resource and exit + if (process_exit) { + break; + } + } + free(evts); + return; +} + +static int rexec_run(int efd, int connfd, char *argv[]) +{ + int pidfd = -1; + int exit_status = EXIT_FAILURE; + + struct rexec_client_event *connevt = rexec_add_event(efd, connfd, -1, rexec_conn_msg); + if (NULL == connevt || rexec_set_nonblock(connfd, 1) != 0) { + // process will exit, fd or mem resource will free by kernel soon + rexec_err("rexec add connfd event failed"); + return exit_status; + } + // 这两个指针只能在当前函数上下文使用，是当前函数栈指针 + connevt->exit_status = &exit_status; + connevt->pidfd = &pidfd; + + rexec_log("Rexec process start run, as proxy of remote %s", argv[1]); + rexec_event_run(efd); + rexec_log("Rexec process %s exit.", argv[1]); + + // clear pidmap file + if (pidfd > 0) { + char path[32] = {0}; + sprintf(path, "%s/%d", REXEC_PIDMAP_PATH, getpid()); + close(pidfd); + remove(path); + } + +end: + close(efd); + return exit_status; +} + +void rexec_create_pidmap_path() +{ + if (access(REXEC_RUN_PATH, F_OK) != 0) { + mkdir(REXEC_RUN_PATH, 0700); + } + mkdir(REXEC_PIDMAP_PATH, 0700); + return; +} + +void rexec_clear_pids() +{ + char path[REXEC_PIDMAP_PATH_LEN] = {0}; + DIR *dir = NULL; + struct dirent *entry; + if (access(REXEC_PIDMAP_PATH, F_OK) != 0) { + rexec_create_pidmap_path(); + return; + } + dir = opendir(REXEC_PIDMAP_PATH); + if (dir == NULL) { + rexec_err("open path:%s failed", REXEC_PIDMAP_PATH); + return; + } + while (entry = readdir(dir)) { + int fd; + + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0 || + strlen(entry->d_name) >= REXEC_PID_LEN) + continue; + + memset(path, 0, sizeof(path)); + sprintf(path, "%s/%s", REXEC_PIDMAP_PATH, entry->d_name); + fd = open(path, O_RDONLY); + if (fd <= 0) { + rexec_err("open pid file:%s failed", path); + continue; + } + if (flock(fd, LOCK_EX|LOCK_NB) != 0) { + close(fd); + continue; + } + close(fd); + if (remove(path) != 0) { + rexec_err("remove unuse pidmap file:%s failed", path); + } + } + + closedir(dir); + return; +} + +#define REXEC_PATH_MAX 4096 +struct rexec_fdinfo { + int fd; + char path[REXEC_PATH_MAX]; + unsigned int perm; + int offset; +}; + +static inline int rexec_is_reg_file(int fd) +{ + if (S_ISREG(rexec_fd_mode(fd))) + return 1; + return 0; +} + +static int rexec_get_fdinfo(struct dirent *fdentry, struct rexec_fdinfo *fdinfo) +{ + char path[32] = {0}; + int ret; + int fd = atoi(fdentry->d_name); + if (fd <= STDERR_FILENO || fd == fileno(rexec_logfile)) + return -1; + if (!rexec_is_reg_file(fd)) + return -1; + sprintf(path, "/proc/self/fd/%s", fdentry->d_name); + ret = readlink(path, fdinfo->path, REXEC_PATH_MAX); + if (ret < 0) { + rexec_err("Get fd:%d link failed.", fd); + return -1; + } + fdinfo->fd = fd; + fdinfo->offset = lseek(fd, 0, SEEK_CUR); + + fdinfo->perm = fcntl(fd, F_GETFL, NULL); + if (fdinfo->perm == -1) { + rexec_err("Get fd:%d flags failed", fd); + return -1; + } + return 0; +} +// 返回一个拼装好的json格式字符串，内存在内部申请好 +// 由调用者释放 +// 内容是本进程所有REG类型文件的信息 +static char *rexec_get_fds_jsonstr() +{ + struct json_object *root = json_object_new_object(); + char *json_str; + int len; + DIR *fddir = NULL; + struct dirent *fdentry; + struct rexec_fdinfo *fdinfo; + if (root == NULL) { + rexec_err("create json-c root failed."); + return NULL; + } + fdinfo = (struct rexec_fdinfo *)malloc(sizeof(struct rexec_fdinfo)); + if (fdinfo == NULL) { + rexec_err("malloc failed."); + goto err_end; + } + + fddir = opendir("/proc/self/fd"); + if (fddir == NULL) { + free(fdinfo); + rexec_err("open path:/proc/self/fd failed"); + goto err_end; + } + + struct json_object *files_arr = json_object_new_array(); + + while (fdentry = readdir(fddir)) { + struct json_object *fd_obj = json_object_new_object(); + struct json_object *item = NULL; + + if (fd_obj == NULL) { + rexec_err("json c new object failed."); + goto json_err; + } + memset(fdinfo, 0, sizeof(struct rexec_fdinfo)); + if (rexec_get_fdinfo(fdentry, fdinfo) != 0) { + json_object_put(fd_obj); + continue; + } + item = json_object_new_int(fdinfo->fd); + json_object_object_add(fd_obj, "Fd", item); + item = json_object_new_string(fdinfo->path); + json_object_object_add(fd_obj, "Path", item); + item = json_object_new_int(fdinfo->perm); + json_object_object_add(fd_obj, "Perm", item); + item = json_object_new_int(fdinfo->offset); + json_object_object_add(fd_obj, "Offset", item); + + json_object_array_add(files_arr, fd_obj); + } + closedir(fddir); + free(fdinfo); + + json_object_object_add(root, "Files", files_arr); + json_str = strdup(json_object_get_string(root)); + json_object_put(root); + + return json_str; + +json_err: + closedir(fddir); + free(fdinfo); +err_end: + json_object_put(root); + return NULL; +} + +// 将rexec进程从parent继承到的匿名pipe继承给远端进程 +static int rexec_pipe_remote_inherit(int efd, int connfd) +{ +#define SELF_FD_PATH "/proc/self/fd" + DIR *fddir = NULL; + struct dirent *fdentry; + struct rexec_msg msg; + mode_t mode; + int pfd[2]; + + fddir = opendir(SELF_FD_PATH); + if (fddir == NULL) { + rexec_err("open path:%s failed", SELF_FD_PATH); + return -1; + } + memset(&msg, 0, sizeof(struct rexec_msg)); + msg.msglen = 0; + msg.pipefd = -1; + msg.msgtype = REXEC_PIPE; + while (fdentry = readdir(fddir)) { + int fd = atoi(fdentry->d_name); + if (fd <= STDERR_FILENO) + continue; + mode = rexec_fd_mode(fd); + if (!S_ISFIFO(mode)) + continue; + rexec_log("inherit pipe fd:%d mode:%o is %s pipe", fd, mode, (!!(mode & S_IRUSR)) ? "read" : "write"); + if (pipe(pfd) == -1) { + rexec_err("failed to create pipe for:%d", fd); + goto err_end; + } + msg.pipefd = fd; + if (!!(mode & S_IRUSR)) { + // inherit read pipe + if (rexec_sendmsg(connfd, (char *)&msg, sizeof(struct rexec_msg), pfd[PIPE_READ]) < 0) { + rexec_err("send read pipe failed, inherit fd:%d", fd); + goto pipe_end; + } + if (rexec_add_event(efd, fd, pfd[PIPE_WRITE], rexec_io) == NULL) { + rexec_err("add read pipe event failed:%d", fd); + goto pipe_end; + } + close(pfd[PIPE_READ]); + } else if (!!(mode & S_IWUSR)) { + if (rexec_sendmsg(connfd, (char *)&msg, sizeof(struct rexec_msg), pfd[PIPE_WRITE]) < 0) { + rexec_err("send write pipe failed, inherit fd:%d", fd); + goto pipe_end; + } + if (rexec_add_event(efd, pfd[PIPE_READ], fd, rexec_io) == NULL) { + rexec_err("add write pipe event failed:%d", fd); + goto pipe_end; + } + close(pfd[PIPE_WRITE]); + } + rexec_log("successed to add pipe fd:%d to remote inherit", fd); + } + closedir(fddir); + return 0; + +pipe_end: + close(pfd[0]); + close(pfd[1]); +err_end: + closedir(fddir); + return -1; +} + +static int rexec_handshake_proc(struct rexec_client_event *evt) +{ + char msg[sizeof(struct rexec_msg) + 1]; + struct rexec_msg *hs = msg; + int ret = read(evt->fd, hs->msg, 1); + if (ret <= 0) { + rexec_err("read from handshake pipe failed, ret:%d err:%d", ret, errno); + return REXEC_EVENT_DEL; + } + hs->msgtype = REXEC_HANDSHAKE; + hs->msglen = 1; + ret = write(evt->outfd, hs, sizeof(struct rexec_msg) + 1); + if (ret < 0) { + rexec_err("send handshake failed, connfd:%d.", evt->outfd); + } + return REXEC_EVENT_OK; +} + +static int rexec_handshake_init(int efd, int connfd) +{ + char *hs_read = getenv("REXEC_HANDSHAKE_RD"); + if (hs_read == NULL) { + rexec_log("handshake not in effect."); + return 0; + } + g_rexec.rexec_hs_fd[PIPE_READ] = atoi(hs_read); + + char *hs_write = getenv("REXEC_HANDSHAKE_WR"); + if (hs_write == NULL) { + rexec_log("handshake not in effect, read:%s", hs_read); + g_rexec.rexec_hs_fd[PIPE_READ] = -1; + return 0; + } + g_rexec.rexec_hs_fd[PIPE_WRITE] = atoi(hs_write); + + if (g_rexec.rexec_hs_fd[PIPE_READ] <= STDERR_FILENO || g_rexec.rexec_hs_fd[PIPE_WRITE] <= STDERR_FILENO) { + rexec_log("handshake invalid fd read:%d write:%d", g_rexec.rexec_hs_fd[PIPE_READ], g_rexec.rexec_hs_fd[PIPE_WRITE]); + goto err_end; + } + if (!S_ISFIFO(rexec_fd_mode(g_rexec.rexec_hs_fd[PIPE_READ])) || !S_ISFIFO(rexec_fd_mode(g_rexec.rexec_hs_fd[PIPE_WRITE]))) { + rexec_err("handshake fd mode not fifo:%d %d", g_rexec.rexec_hs_fd[PIPE_READ], g_rexec.rexec_hs_fd[PIPE_WRITE]); + goto err_end; + } + if (rexec_add_event(efd, g_rexec.rexec_hs_fd[PIPE_READ], connfd, rexec_handshake_proc) == NULL) { + rexec_err("add handshake pipe read fd:%d to epoll failed", g_rexec.rexec_hs_fd[PIPE_READ]); + goto err_end; + } + rexec_log("handshake effect read:%d write:%d", g_rexec.rexec_hs_fd[PIPE_READ], g_rexec.rexec_hs_fd[PIPE_WRITE]); + return 0; +err_end: + g_rexec.rexec_hs_fd[PIPE_READ] = -1; + g_rexec.rexec_hs_fd[PIPE_WRITE] = -1; + return -1; +} + +static int rexec_send_binary_msg(int efd, int argc, char *argv[], int arglen, char *fds_json, int connfd) +{ + struct rexec_msg *pmsg = (struct rexec_msg *)malloc(arglen); + if (pmsg == NULL) { + rexec_err("malloc failed"); + free(fds_json); + return -1; + } + char *bufmsg = pmsg->msg; + memset(pmsg, 0, arglen); + pmsg->msgtype = REXEC_EXEC; + pmsg->argc = argc - 1; // for remote binary's argc is argc-1 + // pmsg->msg is like: "binary"\0"argv[1]"\0"argv[2]"\0"..." + pmsg->msglen = rexec_msg_fill_argv(pmsg->argc, &argv[1], bufmsg); + strcpy(&bufmsg[pmsg->msglen], fds_json); + pmsg->msglen += strlen(fds_json); + free(fds_json); + + // pipefd[0] -- for read; pipefd[1] -- for write. + // rexec stdin --> rstdin[1] ------> rstdin[0] as stdin + // rexec stdout <-- rstdout[0] <------ rstdout[1] as stdout + // rexec stderr <-- rstderr[0] <------ rstderr[1] as stderr + int rstdin[2]; + int rstdout[2]; + int rstderr[2]; + + if (pipe(rstdin) == -1 || pipe(rstdout) == -1 || pipe(rstderr) == -1) { + rexec_err("Rexec create pipe failed."); + goto err_end; + } + pmsg->pipefd = REXEC_STDIN; + if (rexec_sendmsg(connfd, (char *)pmsg, sizeof(struct rexec_msg) + pmsg->msglen, rstdin[0]) < 0) { + rexec_err("Rexec send exec msg failed, errno:%d", errno); + goto err_end; + } + rexec_log("Normal msg send len:%d head:%d", sizeof(struct rexec_msg) + pmsg->msglen, sizeof(struct rexec_msg)); + pmsg->msgtype = REXEC_PIPE; + pmsg->argc = 0; + pmsg->msglen = 0; + pmsg->pipefd = REXEC_STDOUT; + if (rexec_sendmsg(connfd, (char *)pmsg, sizeof(struct rexec_msg), rstdout[1]) < 0) { + rexec_err("Rexec send exec msg failed, errno:%d", errno); + goto err_end; + } + pmsg->pipefd = REXEC_STDERR; + if (rexec_sendmsg(connfd, (char *)pmsg, sizeof(struct rexec_msg), rstderr[1]) < 0) { + rexec_err("Rexec send exec msg failed, errno:%d", errno); + goto err_end; + } + + if (rexec_std_event(efd, rstdin[1], rstdout[0], rstderr[0]) != 0) { + rexec_err("add std event failed"); + goto err_end; + } + free(pmsg); + close(rstdin[0]); + close(rstdout[1]); + close(rstderr[1]); + return 0; +err_end: + free(pmsg); + return -1; +} + +static void *rexec_pipe_proxy_thread(void *arg) +{ + struct rexec_thread_arg *parg = (struct rexec_thread_arg *)arg; + rexec_log("pipe proxy thread run."); + rexec_event_run(parg->efd); + rexec_log("pipe proxy thread run over"); + return NULL; +} + +static void *rexec_conn_thread(void *arg) +{ + struct rexec_thread_arg *parg = (struct rexec_thread_arg *)arg; + + return (void *)rexec_run(parg->efd, parg->connfd, parg->argv); +} + +static void rexec_global_var_init() +{ + memset(&g_rexec, 0, sizeof(g_rexec)); + g_rexec.rexec_hs_fd[PIPE_READ] = -1; + g_rexec.rexec_hs_fd[PIPE_WRITE] = -1; + return; +} + +int main(int argc, char *argv[]) +{ + rexec_log_init(); + rexec_clear_pids(); + + int pipeefd = epoll_create1(0); + int efd = epoll_create1(0); + if (efd == -1 || pipeefd == -1) { + rexec_err("epoll create1 failed, errno:%d.", errno); + return -1; + } + rexec_global_var_init(); + + int connfd = rexec_conn_to_server(); + if (connfd < 0) { + rexec_err("Rexec connect to server failed, errno:%d", errno); + return -1; + } + + if (rexec_handshake_init(efd, connfd) != 0) { + rexec_err("Rexec handshake environment set but get error."); + return -1; + } + rexec_log("Remote exec binary:%s", argv[1]); + /*if (rexec_pipe_remote_inherit(pipeefd, connfd) != 0) { + rexec_err("Rexec pipe remote inherit failed."); + goto err_end; + }*/ + + int arglen = rexec_calc_argv_len(argc - 1, &argv[1]); + if (arglen <= 0) { + rexec_err("argv is invalid."); + return -1; + } + char *fds_json = rexec_get_fds_jsonstr(); + if (fds_json == NULL) { + rexec_err("Get fds info json string failed."); + return -1; + } + arglen += sizeof(struct rexec_msg); + arglen += strlen(fds_json); + arglen = ((arglen / REXEC_MSG_LEN) + 1) * REXEC_MSG_LEN; + if (arglen <= 0) { + rexec_err("invalid arguments length:%d.", arglen); + free(fds_json); + return -1; + } + + if (rexec_send_binary_msg(efd, argc, argv, arglen, fds_json, connfd) != 0) { + rexec_err("send binary information message failed."); + goto err_end; + } + + pthread_t thrd; + pthread_t thrd_conn; + struct rexec_thread_arg targ; + struct rexec_thread_arg connarg; + void *exit_status; + targ.efd = pipeefd; + (void)pthread_create(&thrd, NULL, rexec_pipe_proxy_thread, &targ); + + connarg.efd = efd; + connarg.connfd = connfd; + connarg.argv = argv; + (void)pthread_create(&thrd_conn, NULL, rexec_conn_thread, &connarg); + pthread_join(thrd_conn, (void *)&exit_status); + fclose(rexec_logfile); + exit((int)exit_status); +err_end: + fclose(rexec_logfile); + rexec_logfile = NULL; + return -1; +} diff --git a/qtfs/rexec/rexec.h b/qtfs/rexec/rexec.h new file mode 100644 index 0000000..acd6cca --- /dev/null +++ b/qtfs/rexec/rexec.h @@ -0,0 +1,170 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * qtfs licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Liqiang + * Create: 2023-03-20 + * Description: + *******************************************************************************/ + +#ifndef __REXEC_H__ +#define __REXEC_H__ + +#include +#include +#include + +enum { + PIPE_READ = 0, + PIPE_WRITE, +}; + +enum { + REXEC_EVENT_OK, + REXEC_EVENT_DEL, // del this event + REXEC_EVENT_EXIT, // exit process + REXEC_EVENT_ERR, +}; + +enum { + REXEC_STDIN = 0, + REXEC_STDOUT, + REXEC_STDERR, + REXEC_NONE, +}; + +#define REXEC_MSG_1K 1024 +#define REXEC_MSG_MAX REXEC_MSG_1K * 1024 +#define REXEC_LOG_PATH_LEN 64 + +// rexec client与server之间建联的sock文件路径 +#define REXEC_UDS_CONN "/var/run/rexec/rexec_uds.sock" +#define REXEC_LOCK_PATH "/var/run/rexec/rexec_uds.lock" +#define REXEC_RUN_PATH "/var/run/rexec/" + +enum rexec_msgtype { + REXEC_EXEC = 0x5a5a, // exec process + REXEC_KILL, // kill process + REXEC_PIPE, // client send a pipefd as stdin/out/err to server + REXEC_PIDMAP, // server send remote process's pid to client + REXEC_HANDSHAKE, +}; + +struct rexec_msg { + int msgtype; + // client to server + int argc; + int pipefd; + int msglen; + // server to client + int exit_status; + int pid; // for pidmap + char msg[0]; +}; + +extern FILE *rexec_logfile; +static inline void rexec_log_init() +{ + char deflog[REXEC_LOG_PATH_LEN] = "/dev/null"; + char *logfile = getenv("REXEC_LOG_FILE"); + if (logfile == NULL) { + logfile = deflog; + } else if (strcmp(logfile, "std") == 0) { + rexec_logfile = stderr; + return; + } else { + printf("REXEC_LOG_FILE cannot be set to any value other than std\n"); + } +retry: + rexec_logfile = fopen(logfile, "a"); + if (rexec_logfile == NULL) { + if (strcmp(logfile, "/dev/null") == 0) { + return; + } + // 输入的文件打开失败则回退到无日志模式 + logfile = deflog; + goto retry; + } + return; +} + +// flag: 1--nonblock; 0--block +static inline int rexec_set_nonblock(int fd, int block) +{ + int fflags; + if ((fflags = fcntl(fd, F_GETFL)) < 0) + return -1; + if (block == 0) + fflags &= ~O_NONBLOCK; + else + fflags |= O_NONBLOCK; + if ((fcntl(fd, F_SETFL, fflags)) < 0) + return -1; + return 0; +} + +static inline int rexec_set_inherit(int fd, bool inherit) +{ + int fflags; + if ((fflags = fcntl(fd, F_GETFD)) < 0) + return -1; + if (inherit) + fflags &= ~FD_CLOEXEC; + else + fflags |= FD_CLOEXEC; + if ((fcntl(fd, F_SETFD, fflags)) < 0) + return -1; + return 0; +} + +#define rexec_log(info, ...) \ + if (rexec_logfile != NULL) {\ + time_t t; \ + struct tm p; \ + time(&t); \ + localtime_r(&t, &p); \ + fprintf(rexec_logfile, "[%d/%02d/%02d %02d:%02d:%02d][LOG:%s:%3d]"info"\n", \ + p.tm_year + 1900, p.tm_mon+1, p.tm_mday, \ + p.tm_hour, p.tm_min, p.tm_sec, __func__, __LINE__, ##__VA_ARGS__); \ + } + +#define rexec_log2(info, ...) \ + if (rexec_logfile != NULL) {\ + time_t t; \ + struct tm p; \ + time(&t); \ + localtime_r(&t, &p); \ + fprintf(rexec_logfile, "[%d/%02d/%02d %02d:%02d:%02d][LOG:%s:%3d]"info"\n", \ + p.tm_year + 1900, p.tm_mon+1, p.tm_mday, \ + p.tm_hour, p.tm_min, p.tm_sec, __func__, __LINE__, ##__VA_ARGS__); \ + } + +#define rexec_err(info, ...) \ + if (rexec_logfile != NULL) {\ + time_t t; \ + struct tm p; \ + time(&t); \ + localtime_r(&t, &p); \ + fprintf(rexec_logfile, "[%d/%02d/%02d %02d:%02d:%02d][ERROR:%s:%3d]"info"\n", \ + p.tm_year + 1900, p.tm_mon+1, p.tm_mday, \ + p.tm_hour, p.tm_min, p.tm_sec, __func__, __LINE__, ##__VA_ARGS__); \ + } + +static inline unsigned int rexec_fd_mode(int fd) +{ + struct stat st; + if (fstat(fd, &st) != 0) { + rexec_err("get fd:%d fstat failed, errno:%d", fd, errno); + return 0; + } + return st.st_mode; +} + +#endif + diff --git a/qtfs/rexec/rexec_server.c b/qtfs/rexec/rexec_server.c new file mode 100644 index 0000000..b8768e9 --- /dev/null +++ b/qtfs/rexec/rexec_server.c @@ -0,0 +1,615 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * qtfs licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Liqiang + * Create: 2023-03-20 + * Description: + *******************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rexec_sock.h" +#include "rexec.h" + +static int main_epoll_fd = -1; +FILE *rexec_logfile = NULL; +static GHashTable *child_hash = NULL; +static volatile sig_atomic_t sig_chld_flag = 0; + +#define REXEC_WHITELIST_MAX_ITEMS 256 +struct rexec_white_list_str { + int wl_nums; + char *wl[REXEC_WHITELIST_MAX_ITEMS]; +}; +static struct rexec_white_list_str rexec_wl; + +extern int rexec_shim_entry(int argc, char *argv[]); + +int rexec_hash_insert_direct(GHashTable *table, int key, int value); +int rexec_hash_lookup_direct(GHashTable *table, int key); +void rexec_hash_remove_direct(GHashTable *table, int key); + +struct rexec_event { + int fd; + union { + int pid; + int connfd; + }; + int (*handler)(struct rexec_event *); +}; + +static int rexec_add_event(int efd, int fd, int pid, int (*handler)(struct rexec_event *)) +{ + struct rexec_event *event = (struct rexec_event *)malloc(sizeof(struct rexec_event)); + if (event == NULL) { + rexec_err("malloc failed."); + return -1; + } + event->fd = fd; + event->pid = pid; + event->handler = handler; + struct epoll_event evt; + evt.data.ptr = (void *)event; + evt.events = EPOLLIN; + if (-1 == epoll_ctl(efd, EPOLL_CTL_ADD, event->fd, &evt)) { + rexec_err("epoll ctl add fd:%d event failed.", event->fd); + free(event); + return -1; + } + return 0; +} + +static int rexec_del_event(int efd, struct rexec_event *event) +{ + int ret = epoll_ctl(efd, EPOLL_CTL_DEL, event->fd, NULL); + if (ret != 0) { + rexec_err("failed to delete event fd:%d.", event->fd); + } else { + rexec_log("event fd:%d deleted.", event->fd); + } + close(event->fd); + free(event); + return 0; +} + +static int rexec_event_process_manage(struct rexec_event *event) +{ + struct rexec_msg head; + int ret = recv(event->fd, &head, sizeof(struct rexec_msg), MSG_WAITALL); + if (ret <= 0) { + rexec_log("Event fd:%d recv ret:%d errno:%d, peer rexec closed, kill the associated process:%d.", + event->fd, ret, errno, event->pid); + kill(event->pid, SIGKILL); + return REXEC_EVENT_DEL; + } + rexec_err("Recv msg from client, msgtype:%d msglen:%d argc:%d pipefd:%d", + head.msgtype, head.msglen, head.argc, head.pipefd); + return REXEC_EVENT_OK; +} + +static int rexec_event_handshake(struct rexec_event *event) +{ + int sonpid = 0; + int ret = read(event->fd, &sonpid, sizeof(int)); + if (ret <= 0) { + rexec_err("Rexec read from pipe ret:%d errno:%d", ret, errno); + return REXEC_EVENT_DEL; + } + int connfd = event->connfd; + if (sonpid == -1) { + rexec_err("Handshake recv -1, dont add to process manage"); + close(connfd); + return REXEC_EVENT_DEL; + } + rexec_log("Rexec recv son pid:%d, connfd:%d", sonpid, connfd); + + rexec_hash_insert_direct(child_hash, sonpid, connfd); + rexec_add_event(main_epoll_fd, connfd, sonpid, rexec_event_process_manage); + + // 成功后同样要删除这个pipe监听事件，删除时会close掉fd + return REXEC_EVENT_DEL; +} + +static int rexec_dup_pipefd(int fd, int pipefd) +{ + int dupfd; + if (fd == -1) + return 0; + dupfd = dup2(fd, pipefd); + if (dupfd != pipefd) { + rexec_err("failed to dup pipefd:%d", pipefd); + return -1; + } + close(fd); + return 0; +} + +// argv list: [0]binary,[1]-f,[2]*json_str,[3]arg1,[4]arg2,... +static int rexec_parse_argv(int argc, char *argv_str, char *argv[]) +{ + int offset = 0; + for (int i = 0; i < argc; i++) { + argv[i] = &argv_str[offset]; + offset += strlen(argv[i]) + 1; + } + argv[argc] = NULL; + return offset; +} + +static inline void rexec_clear_string_tail(char *str, int len) +{ + while (len >= 0 && str[len] < 0x20) { + str[len] = '\0'; + len--; + } + return; +} + +#define REXEC_WHITELIST_FILE "/etc/rexec/whitelist" +static int rexec_whitelist_build(struct rexec_white_list_str *wl) +{ + if (access(REXEC_WHITELIST_FILE, F_OK) != 0) { + rexec_err("Please configure the white list(%s).", REXEC_WHITELIST_FILE); + return -1; + } + + wl->wl_nums = 0; + memset(wl->wl, 0, sizeof(uintptr_t) * REXEC_WHITELIST_MAX_ITEMS); +#define MAX_CMD_LEN 256 + char cmd[MAX_CMD_LEN]; + FILE *fwl = fopen(REXEC_WHITELIST_FILE, "r"); + if (fwl == NULL) { + rexec_err("open white list file:%s failed.", REXEC_WHITELIST_FILE); + return -1; + } + struct stat stats; + int ret = fstat(fileno(fwl), &stats); + if (ret != 0) { + rexec_err("fstat white list file:%s failed.", REXEC_WHITELIST_FILE); + goto err_end; + } + if (stats.st_mode & 0777 != 0400) { + rexec_err("white list file:%s permissions(%o) error, must be read-only(0400)", stats.st_mode, REXEC_WHITELIST_FILE); + goto err_end; + } + while (!feof(fwl) && wl->wl_nums < REXEC_WHITELIST_MAX_ITEMS) { + int len; + char *fstr; + memset(cmd, 0, MAX_CMD_LEN); + fstr = fgets(cmd, MAX_CMD_LEN - 1, fwl); + if (fstr == NULL) + continue; + rexec_clear_string_tail(cmd, strlen(cmd)); + len = strlen(cmd); + fstr = (char *)malloc(len + 1); + if (fstr == NULL) { + rexec_err("Malloc failed"); + goto err_end; + } + memset(fstr, 0, len + 1); + memcpy(fstr, cmd, len); + wl->wl[wl->wl_nums] = fstr; + wl->wl_nums++; + rexec_log("Cmd:<%s> added to white list.", cmd); + } + fclose(fwl); + if (wl->wl_nums == 0) { + rexec_err("White list file [%s] has no valid content.", REXEC_WHITELIST_FILE); + return -1; + } + return 0; + +err_end: + for (int i = 0; i < wl->wl_nums; i++) { + free(wl->wl[i]); + } + fclose(fwl); + return -1; +} + +static void rexec_white_list_free(struct rexec_white_list_str *wl) +{ + for (int i = 0; i < wl->wl_nums; i++) { + free(wl->wl[i]); + } + return; +} + +static int rexec_whitelist_check(char *binary) +{ + // white list file not exist, always ok + if (access(REXEC_WHITELIST_FILE, F_OK) != 0) + return -1; + for (int i = 0; i < rexec_wl.wl_nums; i++) { + if (strlen(binary) == strlen(rexec_wl.wl[i]) && strcmp(binary, rexec_wl.wl[i]) == 0) + return 0; + } + return -1; +} + +#define IS_VALID_FD(fd) (fd > STDERR_FILENO) +static void handle_sig_chld(void) +{ + int status; + pid_t pid; + while ((pid = waitpid(0, &status, WNOHANG)) > 0) { + int exit_status = status; + if (WIFEXITED(status)) { + exit_status = WEXITSTATUS(status); + } else if (WIFSIGNALED(status)) { + exit_status = WTERMSIG(status) + 128; + } + int connfd = rexec_hash_lookup_direct(child_hash, pid); + if (IS_VALID_FD(connfd)) { + struct rexec_msg head; + head.msgtype = REXEC_KILL; + head.msglen = 0; + head.exit_status = exit_status; + rexec_sendmsg(connfd, (char *)&head, sizeof(struct rexec_msg), -1); + rexec_hash_remove_direct(child_hash, pid); + // don't close connfd + } + } + return; +} + +static void rexec_server_sig_chld(int num) +{ + __sync_fetch_and_add(&sig_chld_flag, 1); + return; +} + +static void rexec_server_sig_pipe(int signum) +{ + return; +} + +#define REXEC_MSG_NORMAL (1 << 3) +#define REXEC_MSG_OVER 0xf +static int rexec_start_new_process(int newconnfd) +{ + int ret; + int pipefd[2]; + if (pipe(pipefd) == -1) { + rexec_err("pipe syscall error, errno:%d", errno); + return -1; + } + // handshake阶段，联合体里面记录newconnfd + // 等到handshake成功后，新的事件监听这个newconnfd，联合体改为记录son pid + rexec_add_event(main_epoll_fd, pipefd[PIPE_READ], newconnfd, rexec_event_handshake); + + int pid = fork(); + // parent + if (pid != 0) { + close(pipefd[PIPE_WRITE]); + return 0; + } + // son + close(pipefd[PIPE_READ]); + + struct rexec_msg head; + int argc; + char *msgbuf = NULL; + char msg_bit = 0; + while (msg_bit != REXEC_MSG_OVER) { + int scmfd = -1; + int len = sizeof(struct rexec_msg); + memset(&head, 0, sizeof(struct rexec_msg)); + scmfd = -1; + ret = rexec_recvmsg(newconnfd, (char *)&head, len, &scmfd, MSG_WAITALL); + if (ret <= 0) { + rexec_log("recvmsg ret:%d, errno:%d", ret, errno); + goto err_to_parent; + } + // 将管道与自己的标准输入输出关联 + if (rexec_dup_pipefd(scmfd, head.pipefd) != 0) { + rexec_err("dup scm:%d pipefd:%d failed", scmfd, head.pipefd); + goto err_to_parent; + } + if (head.pipefd >= REXEC_STDIN && head.pipefd <= REXEC_STDERR) { + msg_bit |= (1 << (head.pipefd - REXEC_STDIN)); + } + if (head.msglen == 0) + continue; + // 普通消息，代码暂时没考虑多个普通消息的，先直接过滤 + if (msgbuf != NULL || head.msgtype != REXEC_EXEC) { + rexec_err("not support multi normal msg or msgtype:%d msglen:%d invalid", head.msgtype, head.msglen); + continue; + } + msg_bit |= REXEC_MSG_NORMAL; + // exec msg + rexec_log("Exec msgtype:0x%x msglen:%d argc:%d stdno:%d", + head.msgtype, head.msglen, head.argc, head.pipefd); + argc = head.argc; + if (head.msglen > REXEC_MSG_MAX || argc > REXEC_MSG_MAX / sizeof(uintptr_t) || + head.msglen <= 0 || argc < 0) { + rexec_err("msg len:%d or argc:%d is too large", head.msglen, argc); + goto err_to_parent; + } + msgbuf = (char *)malloc(head.msglen + 1); + if (msgbuf == NULL) { + rexec_err("malloc failed"); + goto err_to_parent; + } + memset(msgbuf, 0, head.msglen + 1); + ret = recv(newconnfd, msgbuf, head.msglen, MSG_WAITALL); + if (ret <= 0) { + rexec_err("recv failed, ret:%d errno:%d", ret, errno); + goto err_free; + } + rexec_log("recv normal msg len:%d headlen:%d real recv:%d msg:%s", + head.msglen, sizeof(struct rexec_msg), ret, msgbuf); + } + + // msg is like: "binary"\0"argv[1]"\0"argv[2]"\0"..." + char *binary = msgbuf; + if (rexec_whitelist_check(binary) != 0) { + rexec_err("Cmd:<%s> not in white list.", binary); + goto err_free; + } + + char *ack; + int mypid = getpid(); + char msg[sizeof(struct rexec_msg) + 1]; + struct rexec_msg *pm = msg; + pm->msgtype = REXEC_PIDMAP; + pm->msglen = 0; + pm->pid = mypid; + ret = write(newconnfd, pm, sizeof(struct rexec_msg)); + if (ret <= 0) { + rexec_err("Rexec send son pid:%d to client failed, ret:%d errno:%d", mypid, ret, errno); + } else { +retry: + rexec_log("Waiting for rexec client handshake..."); + ret = read(newconnfd, pm, sizeof(struct rexec_msg) + 1); + if (ret <= 0) { + rexec_err("Recv handshake failed, ret:%d err:%d", ret, errno); + goto err_to_parent; + } + if (pm->msgtype != REXEC_HANDSHAKE) { + rexec_err("Recv unexpected msg:%d", pm->msgtype); + goto retry; + } + ack = pm->msg; + if (*ack != '1') { + rexec_err("recv error handshake ack from client:%c, exit now", *ack); + goto err_to_parent; + } + } + // 写会PID必须放在基于newconnfd接收完所有消息之后， + // 后面newconnfd的控制权交回父进程rexec server服务进程 + if (write(pipefd[PIPE_WRITE], &mypid, sizeof(int)) <= 0) { + rexec_err("write pid to parent failed, pipefd:%d.", pipefd[PIPE_WRITE]); + } + // 子进程不再使用pipe write和connfd + close(pipefd[PIPE_WRITE]); + close(newconnfd); + + rexec_log("handshake over normaly, continue to exec new process:%s.", binary); + + // rexec_shim_entry argv like: + // argv[0]: binary + // argv[1]: -f + // argv[2]: *json_str + // argv[3]: param list 1 + // argv[4]: ... + char **argv = (char **)malloc(sizeof(uintptr_t) * (argc + 3)); + if (argv == NULL) { + rexec_err("malloc failed, argc:%d.", argc); + goto err_free; + } + int offset = rexec_parse_argv(argc, msgbuf, &argv[2]); + argv[0] = "-f"; + argv[1] = &msgbuf[offset]; + + rexec_log("Parse argv result argc:%d", argc); + for (int i = 2; i < argc + 2; i++) { + rexec_log(" argv[%d]:%s", i - 2, argv[i]); + } + ret = rexec_shim_entry(argc + 2, argv); + perror("rexec shim entry"); + +err_free: + free(msgbuf); + +err_to_parent: + do { + int errpid = -1; + write(pipefd[PIPE_WRITE], &errpid, sizeof(int)); + } while (0); + + exit(0); +} + +// 道生一 +static int rexec_event_new_process(struct rexec_event *event) +{ + int newconnfd = rexec_sock_step_accept(event->fd); + if (newconnfd < 0) { + rexec_err("Accept failed, ret:%d errno:%d", newconnfd, errno); + return REXEC_EVENT_ERR; + } + // 主进程只负责接收新链接，基于newconnfd的新消息由子进程自己去接收，但是最 + // 后父进程要进入监听此链接的状态，是为了联动kill（对端杀死client进程则本端 + // 也杀死，或者本端杀死子进程后消息通知对端也杀死） + // 这个fd不能同时被父子进程监听，所以先建立一个pipe，等子进程完全接收完 + // 初始消息后，通过pipe告知父进程再由父进程接管newconnfd，在这之前，父进程 + // 监听pipe的read端 + // 白名单也在子进程里做，在fork之后，rexec代码控制范围 + rexec_log("Start new process new conn fd:%d", newconnfd); + rexec_start_new_process(newconnfd); + return REXEC_EVENT_OK; +} + +static void rexec_server_mainloop() +{ +#define REXEC_MAX_EVENTS 16 + main_epoll_fd = epoll_create1(0); + if (main_epoll_fd == -1) { + rexec_err("epoll create1 failed, errno:%d.", errno); + return; + } + if (rexec_set_inherit(main_epoll_fd, false) < 0) { + rexec_err("epoll fd set inherit to false failed."); + } + struct rexec_conn_arg ser = { + .cs = REXEC_SOCK_SERVER, + .udstype = SOCK_STREAM, + }; + strncpy(ser.sun_path, REXEC_UDS_CONN, strlen(REXEC_UDS_CONN) + 1); + int buildret = rexec_build_unix_connection(&ser); + if (buildret != 0) { + rexec_err("faild to build main sock:%d errno:%d", buildret, errno); + close(main_epoll_fd); + return; + } + if (chmod(REXEC_UDS_CONN, 0600) < 0) { + rexec_err("failed to set uds sock file mode:%s errno:%d", REXEC_UDS_CONN, errno); + close(main_epoll_fd); + close(ser.sockfd); + return; + } + if (rexec_set_inherit(ser.sockfd, false) < 0) { + rexec_err("cs conn fd fd set inherit to false failed."); + } + rexec_add_event(main_epoll_fd, ser.sockfd, 0, rexec_event_new_process); + + struct epoll_event *evts = calloc(REXEC_MAX_EVENTS, sizeof(struct epoll_event)); + if (evts == NULL) { + rexec_err("init calloc evts failed."); + goto end; + } + while (1) { + if (sig_chld_flag != 0 && __sync_fetch_and_sub(&sig_chld_flag, 1) != 0) { + handle_sig_chld(); + } + int n = epoll_wait(main_epoll_fd, evts, REXEC_MAX_EVENTS, 1000); + if (n == 0) + continue; + if (n < 0) { + rexec_err("epoll wait return errcode:%d", n); + continue; + } + rexec_log("epoll wait trigger %d events.", n); + for (int i = 0; i < n; i++) { + struct rexec_event *event = (struct rexec_event *)evts[i].data.ptr; + int ret = event->handler(event); + if (ret == REXEC_EVENT_DEL) + rexec_del_event(main_epoll_fd, event); + } + } + free(evts); + +end: + close(main_epoll_fd); + main_epoll_fd = -1; + close(ser.sockfd); + ser.sockfd = -1; + return; +} + +// hash map for child pid and conn fd +int rexec_pid_hashmap_init(GHashTable **table) +{ + *table = g_hash_table_new(g_direct_hash, g_direct_equal); + if (*table == NULL) { + rexec_err("Init child pid hashmap failed."); + return -1; + } + return 0; +} + +void rexec_pid_hashmap_destroy(GHashTable *table) +{ + g_hash_table_destroy(table); + return; +} + +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-to-int-cast" +int rexec_hash_insert_direct(GHashTable *table, int key, int value) +{ + if (g_hash_table_insert(table, (gpointer)key, (gpointer)value) == 0) { + rexec_err("Hash table key:%d value:%d is already exist, update it.", key, value); + } + return 0; +} + +int rexec_hash_lookup_direct(GHashTable *table, int key) +{ + return (int)g_hash_table_lookup(table, (gpointer)key); +} + +void rexec_hash_remove_direct(GHashTable *table, int key) +{ + g_hash_table_remove(table, (gpointer)key); + return; +} +#pragma GCC diagnostic pop + +int check_socket_lock(void) +{ + int lock_fd = open(REXEC_LOCK_PATH, O_RDONLY | O_CREAT, 0600); + if (lock_fd == -1) + return -EINVAL; + + return flock(lock_fd, LOCK_EX | LOCK_NB); +} + +int main(int argc, char *argv[]) +{ + mode_t newmask = 0077; + rexec_log("Change umask from:%o to %o", umask(newmask), newmask); + rexec_log_init(); + signal(SIGCHLD, rexec_server_sig_chld); + signal(SIGPIPE, rexec_server_sig_pipe); + if (rexec_whitelist_build(&rexec_wl) != 0) { + return -1; + } + if (access(REXEC_RUN_PATH, F_OK) != 0) { + mkdir(REXEC_RUN_PATH, 0700); + } + + if (check_socket_lock() < 0) { + return -1; + } + + if (rexec_pid_hashmap_init(&child_hash) != 0) { + rexec_white_list_free(&rexec_wl); + return -1; + } + rexec_server_mainloop(); + rexec_pid_hashmap_destroy(child_hash); + fclose(rexec_logfile); + rexec_logfile = NULL; + return 0; +} + diff --git a/qtfs/rexec/rexec_shim.c b/qtfs/rexec/rexec_shim.c new file mode 100644 index 0000000..5bd8a19 --- /dev/null +++ b/qtfs/rexec/rexec_shim.c @@ -0,0 +1,157 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * qtfs licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Liqiang + * Create: 2023-03-20 + * Description: + *******************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dirent.h" +#include "rexec.h" + +#define rshim_log rexec_log +#define rshim_err rexec_err + +void rshim_close_all_fd() +{ + DIR *dir = NULL; + struct dirent *entry; + dir = opendir("/proc/self/fd/"); + if (dir == NULL) { + rshim_err("open path:/proc/self/fd/ failed"); + return; + } + while (entry = readdir(dir)) { + int fd = atoi(entry->d_name); + if (fd <= 2 || S_ISFIFO(rexec_fd_mode(fd))) + continue; + close(fd); + } + closedir(dir); + return; +} + +int rshim_get_file_size(char *file) +{ + int size = 0; + FILE *f = fopen(file, "rb"); + if (f == NULL) { + rshim_err("File:%s fopen failed.", file); + return -1; + } + fseek(f, 0, SEEK_END); + size = ftell(f); + fclose(f); + return size; +} + +void rshim_reg_file_open(int fd_target, const char *path, int perm, int offset) +{ + int fd = open(path, perm); + int fd2 = -1; + if (fd < 0) { + rshim_err("Open file:%s failed, fd:%d errno:%d", path, fd, errno); + return; + } + if (fd != fd_target) { + fd2 = dup2(fd, fd_target); + if (fd2 != fd_target) { + rshim_err("Failed to open file:%s by fd:%d", path, fd_target); + close(fd2); + close(fd); + return; + } + close(fd); + } + int off = lseek(fd_target, offset, SEEK_SET); + if (off < 0) { + rshim_err("Failed to set offset:%d to file:%s, fd:%d, fd2:%d", offset, path, fd, fd2); + return; + } + rshim_log("Successed to set offset:%d to file:%s, fd:%d, fd2:%d", offset, path, fd, fd2); + return; +} + +void rshim_reg_file_resume(const char * const json_buf) +{ + struct json_object *obj_files; + struct json_object *obj_file; + struct json_object *obj_fd; + struct json_object *obj_path; + struct json_object *obj_perm; + struct json_object *obj_offset; + int fd, perm, offset; + const char *path = NULL; + int curfd = 3; // begin from 3 + struct json_object *fd_json = json_tokener_parse(json_buf); + if (fd_json == NULL) { + fprintf(stderr, "parse json error\n"); + return; + } + obj_files = json_object_object_get(fd_json, "Files"); + int arraylen = json_object_array_length(obj_files); + for (int i=0; i< arraylen; i++){ + obj_file = json_object_array_get_idx(obj_files, i); + obj_fd = json_object_object_get(obj_file, "Fd"); + fd = json_object_get_int(obj_fd); + obj_path = json_object_object_get(obj_file, "Path"); + path = json_object_get_string(obj_path); + obj_perm = json_object_object_get(obj_file, "Perm"); + perm = json_object_get_int(obj_perm); + obj_offset = json_object_object_get(obj_file, "Offset"); + offset = json_object_get_int(obj_offset); + rshim_log("Get file from json fd:%d path:%s perm:%d offset:%d", + fd, path, perm, offset); + rshim_reg_file_open(fd, path, perm, offset); + } + + json_object_put(fd_json); + return; +} + +/* + param list: + 1) -f xxx.json binary param1 param2 ... + 2) binary param1 param2... +*/ +int rexec_shim_entry(int argc, char *argv[]) +{ + char *json_str = NULL; + char **newarg = NULL; + + if (strcmp(argv[0], "-f") == 0) { + json_str = argv[1]; + newarg = &argv[2]; + } else { + newarg = argv; + } + + rshim_close_all_fd(); + + rshim_log("Get json str:%s", json_str); + + rshim_reg_file_resume(json_str); + execvp(newarg[0], newarg); + perror("execvp failed."); + + return -1; +} + diff --git a/qtfs/rexec/rexec_sock.c b/qtfs/rexec/rexec_sock.c new file mode 100644 index 0000000..e8750c5 --- /dev/null +++ b/qtfs/rexec/rexec_sock.c @@ -0,0 +1,177 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * qtfs licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Liqiang + * Create: 2023-03-20 + * Description: + *******************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rexec.h" +#include "rexec_sock.h" + +int rexec_build_unix_connection(struct rexec_conn_arg *arg) +{ + const int sock_max_conn = 5; + if (arg->cs > REXEC_SOCK_SERVER) { + rexec_err("cs type %d is error.", arg->cs); + return -1; + } + struct sockaddr_un sock_addr = { + .sun_family = AF_UNIX, + }; + int sock_fd = socket(AF_UNIX, arg->udstype, 0); + + if (sock_fd < 0) { + rexec_err("As %s failed, socket fd: %d, errno:%d.", + (arg->cs == REXEC_SOCK_CLIENT) ? "client" : "server", + sock_fd, errno); + return -1; + } + strncpy(sock_addr.sun_path, arg->sun_path, sizeof(sock_addr.sun_path)); + arg->sockfd = sock_fd; + + if (arg->cs == REXEC_SOCK_SERVER) { + unlink(sock_addr.sun_path); + if (bind(sock_fd, (struct sockaddr *)&sock_addr, sizeof(sock_addr)) < 0) { + rexec_err("As server failed, bind error, errno:%d.", + errno); + goto close_and_return; + } + if (listen(sock_fd, sock_max_conn) < 0) { + rexec_err("As server listen failed, errno:%d.", errno); + goto close_and_return; + } + } else { + if (connect(arg->sockfd, (struct sockaddr *)&sock_addr, sizeof(struct sockaddr_un)) < 0) { + goto close_and_return; + } + arg->connfd = sock_fd; + rexec_log("Connect to server successed, sun path:%s", arg->sun_path); + } + + return 0; +close_and_return: + rexec_log("close sockfd:%d and return", sock_fd); + close(sock_fd); + return -1; + +} + +int rexec_sock_step_accept(int sock_fd) +{ + struct sockaddr_un un_addr; + socklen_t len = sizeof(struct sockaddr_un); + int connfd; + connfd = accept(sock_fd, (struct sockaddr *)&un_addr, &len); + if (connfd < 0) { + rexec_err("Accept error:%d, errno:%d.", connfd, errno); + return connfd; + } + rexec_log("Accept success, sun path:%s", un_addr.sun_path); + return connfd; +} + +// send a normal msg +// or a SCM_RIGHTS fd +// or a normal msg and a SCM_RIGHTS fd +int rexec_sendmsg(int sockfd, char *msgbuf, int msglen, int scmfd) +{ + struct msghdr msg; + struct cmsghdr *cmsg = NULL; + struct iovec iov; + char buf[CMSG_SPACE(sizeof(scmfd))]; + int ret; + + memset(&msg, 0, sizeof(msg)); + iov.iov_base = (void *)msgbuf; + iov.iov_len = msglen; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_name = NULL; + msg.msg_namelen = 0; + + msg.msg_control = buf; + msg.msg_controllen = sizeof(buf); + if (scmfd > 0) { + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(scmfd)); + /* Initialize the payload: */ + memcpy(CMSG_DATA(cmsg), &scmfd, sizeof(scmfd)); + msg.msg_controllen = cmsg->cmsg_len; + } else { + msg.msg_controllen = 0; + } + if ((ret = sendmsg(sockfd, &msg, 0)) != iov.iov_len) { + return ret; + } + return ret; +} + +int rexec_recvmsg(int sockfd, char *msgbuf, int len, int *scmfd, int flags) +{ + struct iovec iov; + struct msghdr msg; + int fd = -1; + struct cmsghdr *cmsg; + char buf[CMSG_SPACE(sizeof(fd))]; + + /* send at least one char */ + memset(&msg, 0, sizeof(msg)); + iov.iov_base = msgbuf; + iov.iov_len = len; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_name = NULL; + msg.msg_namelen = 0; + + msg.msg_control = buf; + msg.msg_controllen = sizeof(buf); + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(fd)); + /* Initialize the payload: */ + memcpy(CMSG_DATA(cmsg), &fd, sizeof(fd)); + msg.msg_controllen = cmsg->cmsg_len; + + int ret = recvmsg(sockfd, &msg, flags); + if (ret < 0) + return ret; + + cmsg = CMSG_FIRSTHDR(&msg); + if (cmsg != NULL) { + memcpy(&fd, CMSG_DATA(cmsg), sizeof(fd)); + *scmfd = fd; + } + return ret; +} + + diff --git a/qtfs/rexec/rexec_sock.h b/qtfs/rexec/rexec_sock.h new file mode 100644 index 0000000..c61d6bc --- /dev/null +++ b/qtfs/rexec/rexec_sock.h @@ -0,0 +1,40 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * qtfs licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Liqiang + * Create: 2023-03-20 + * Description: + *******************************************************************************/ + +#ifndef __REXEC_SOCK_H__ +#define __REXEC_SOCK_H__ + +enum { + REXEC_SOCK_CLIENT = 1, + REXEC_SOCK_SERVER, +}; + +#define UDS_SUN_PATH_LEN 108 +struct rexec_conn_arg { + int cs; // client(1) or server(2) + + int udstype; // DGRAM or STREAM + char sun_path[UDS_SUN_PATH_LEN]; + int sockfd; + int connfd; +}; + +int rexec_sock_step_accept(int sock_fd); +int rexec_build_unix_connection(struct rexec_conn_arg *arg); +int rexec_sendmsg(int sockfd, char *msgbuf, int msglen, int scmfd); +int rexec_recvmsg(int sockfd, char *msgbuf, int msglen, int *scmfd, int flags); + +#endif + diff --git a/qtfs/test/cgroup.go b/qtfs/test/cgroup.go new file mode 100644 index 0000000..fcc62ae --- /dev/null +++ b/qtfs/test/cgroup.go @@ -0,0 +1,53 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "sync" + "time" + "strconv" +) + +func do_watch(dir, prefix string) { + entries, err := ioutil.ReadDir(dir) + if err != nil { + fmt.Printf("read cgroup dir(%s) failed: %s\n", dir, err.Error()) + return + } + + for _, entry := range entries { + if entry.IsDir() { + entryPath := path.Join(dir, entry.Name()) + prefix = prefix + " " + do_watch(entryPath, prefix) + } else { + filePath := path.Join(dir, entry.Name()) + file, err := os.Open(filePath) + if err == nil { + file.Close() + } + } + } +} + +func watch(dir, prefix string, wg *sync.WaitGroup) { + do_watch(dir, prefix) + wg.Done() +} + +func main() { + var wg sync.WaitGroup + begin := time.Now() + threads, _ := strconv.Atoi(os.Args[2]) + fmt.Printf("watch %s\n", os.Args[1]) + for i := 0; i < threads; i++ { + wg.Add(1) + go watch(os.Args[1], "", &wg) + fmt.Printf("Thread run %d\n", i) + } + wg.Wait() + dlt := time.Since(begin) + fmt.Printf("All thread over, %d threads cost time:%v\n", threads, dlt) +} diff --git a/qtfs/test/pjdfstest/README.md b/qtfs/test/pjdfstest/README.md new file mode 100644 index 0000000..6272110 --- /dev/null +++ b/qtfs/test/pjdfstest/README.md @@ -0,0 +1,30 @@ +# 开源文件系统测试集pjdfstest + +pjdfstest是一个POSIX系统接口的测试套，用于进行文件系统接口兼容性测试，相关说明及源码可见其开源链接： + +[pjdfstest](https://github.com/pjd/pjdfstest) + +### 部署 + +```bash +# qtfs仍有部分接口兼容未实现，所以pjdfstest用例进行部分裁剪，请使用下述源码进行验证 +$ git clone https://gitee.com/anar/pjdfstest.git + +$ cd pjdfstest + +# pjdfstest源码编译，生成prove二进制 +$ autoreconf-ifs + +$ ./configure + +$ make pjdfstest + +``` + +### 测试 + +* 进入挂载qtfs文件系统的目录 + +* 执行prove -rv ${pjdfstest文件所在目录} + +根据执行结果确定测试是否通过。 diff --git a/qtfs/test/qtfs_test/test_fifo_block.sh b/qtfs/test/qtfs_test/test_fifo_block.sh new file mode 100644 index 0000000..5a1af19 --- /dev/null +++ b/qtfs/test/qtfs_test/test_fifo_block.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# 检查参数 +if [ $# -ne 2 ]; then + echo "Usage: $0 " + echo " thread_count: how many test thread to create." + echo " path: qtfs path, script will create fifo in this path and test." + exit 1 +fi + +# 保存参数 +thread_count=$1 +path=$2 + +# 创建fifo文件 +for i in $(seq 1 $thread_count); do + mkfifo "$path/test_fifo_block_$i" +done + +# 启动线程 +for i in $(seq 1 $thread_count); do + ( + # 读取fifo文件 + read line < "$path/test_fifo_block_$i" + echo "Thread $i read fifo: $line" + # 删除fifo文件 + rm "$path/test_fifo_block_$i" + ) & +done + +# 等待所有线程结束 +wait \ No newline at end of file diff --git a/qtfs/test/qtfs_test/test_handle_open.rs b/qtfs/test/qtfs_test/test_handle_open.rs new file mode 100644 index 0000000..75f6305 --- /dev/null +++ b/qtfs/test/qtfs_test/test_handle_open.rs @@ -0,0 +1,53 @@ +use std::io::{Write, Read}; +use std::net::TcpStream; +use std::mem; +use std::env; + +#[repr(C, packed)] +struct Qtreqopen { + _type: u32, + err: u32, + seq_num: u64, + len: usize, + flags: u64, + mode: u32, + path: [u8; 4096], +} + +fn main() -> std::io::Result<()> { + let argv: Vec = env::args().collect(); + let addr: String = argv[1].parse().expect("please input addr(ip:port)"); + let mut stream = TcpStream::connect(addr).unwrap(); + let mut req = Qtreqopen { + _type: 2, + err: 0, + seq_num: 0, + len: 0, + flags: 0, + mode: 0, + path: [0; 4096], + }; + if argv.len() == 3 { + let s: String = argv[2].parse().expect("Please input path to open by arg 2"); + req.path[..s.len()].copy_from_slice(s.as_bytes()); + req.len = s.len() + 12; + } else { + // send an err packet + req.path = [1; 4096]; + req.len = 4108; + } + let bytes = unsafe { + let ptr = &req as *const Qtreqopen as *const u8; + std::slice::from_raw_parts(ptr, mem::size_of::()) + }; + stream.write_all(&bytes[..24+req.len]).expect("failed to write to socket stream"); + // recv rsp + let mut datain: [u8; 32] = [0; 32]; + stream.read(&mut datain).unwrap(); + if datain[28] == 1 || datain[4] == 1 { + println!("Open failed!"); + } else { + println!("Open successed."); + } + Ok(()) +} \ No newline at end of file diff --git a/qtfs/test/rexec_test/test_rexec_pipe_inherit.c b/qtfs/test/rexec_test/test_rexec_pipe_inherit.c new file mode 100644 index 0000000..de85fc2 --- /dev/null +++ b/qtfs/test/rexec_test/test_rexec_pipe_inherit.c @@ -0,0 +1,23 @@ +#include +#include + +int main(int argc, char *argv[]) +{ + int fd[2]; + pipe(fd); + int pid = fork(); + if (pid == 0) { + // child + char fdstr[16] = {0}; + sprintf(fdstr, "%d", fd[0]); + char *argvchild[] = {"rexec", argv[1], fdstr, NULL}; + close(fd[1]); + execv("/usr/bin/rexec", argvchild); + perror("execv"); + } + close(fd[0]); + sleep(1); + write(fd[1], "hello", 5); + return 0; +} + diff --git a/qtfs/test/rexec_test/test_rexec_pipe_inherit_remote.c b/qtfs/test/rexec_test/test_rexec_pipe_inherit_remote.c new file mode 100644 index 0000000..91e183b --- /dev/null +++ b/qtfs/test/rexec_test/test_rexec_pipe_inherit_remote.c @@ -0,0 +1,13 @@ +#include +#include +#include + +int main(int argc, char *argv[]) +{ + char buf[16] = {0}; + int fd = atoi(argv[1]); + int ret = read(fd, buf, 16); + printf("read from pipe fd:%d string:%s ret:%d errno:%d\n", fd, buf, ret, errno); + return 0; +} + diff --git a/qtfs/test/scmright_test/README.md b/qtfs/test/scmright_test/README.md new file mode 100644 index 0000000..3bc13e4 --- /dev/null +++ b/qtfs/test/scmright_test/README.md @@ -0,0 +1,39 @@ +# 编译： +```bash +make clean +make +``` + +# 测试步骤： +1. 在server上运行 +```bash +cd /home +tar xzvf scmright_test.tar.gz +cd scmright_test +make clean;make +LD_LIBRARY_PATH=. ./server scm.sock +``` + +2. 在client上运行 +```bash +cd /home +tar xzvf scmright_test.tar.gz +cd scmright_test +make clean;make +LD_PRELOAD=/usr/lib64/libudsproxy.so LD_LIBRARY_PATH=. ./client scm.sock test.log +or +LD_PRELOAD=/usr/lib64/libudsproxy.so LD_LIBRARY_PATH=. ./client scm.sock +``` + +3. 往client端发送消息进行测试 +```bash +Input message to send:1234 +Input message to send:abcd +Input message to send:quit +input quit will exit client and server +``` + +4. 确认server是否接收到消息或确认test.log是否写入 +```bash +cat test.log | grep 1234 +``` \ No newline at end of file diff --git a/qtfs/test/scmright_test/client.c b/qtfs/test/scmright_test/client.c new file mode 100644 index 0000000..d27b6f4 --- /dev/null +++ b/qtfs/test/scmright_test/client.c @@ -0,0 +1,81 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "uds.h" + +int main(int argc, char **argv) +{ + if (argc < 2) { + printf("Usage: %s pathtosock [pathtofile]\n", argv[0]); + return -EINVAL; + } + char *path = argv[1]; + char *testfile = NULL; + if (argc == 3) { + testfile = argv[2]; + } + int fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) { + printf("Fail to create socket: %s\n", strerror(errno)); + return -1; + } + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + memcpy(addr.sun_path, path, strlen(path) + 1); + printf("connect to %s\n", addr.sun_path); + int ret = connect(fd, (struct sockaddr*)(&addr), sizeof(addr)); + if (ret < 0) { + printf("Fail to connect: %s\n", strerror(errno)); + close(fd); + return -1; + } + unlink(testfile); + int share_fd; + int pipefd[2]; + char buffer[1024]; + if (testfile == NULL) { + printf("Into PIPE MODE: open pipe\n"); + if (pipe(pipefd) == -1) { + fprintf(stderr, "Pipe failed\n"); + return -1; + } + share_fd = pipefd[1]; + } else { + printf("Into REGULAR FILE MODE: open file %s\n", testfile); + share_fd = open(testfile, O_RDWR | O_CREAT); + } + + if (share_fd < 0) { + printf("open failed, %s\n", strerror(errno)); + return -1; + } + printf("send share_fd[%d] to peer\n", share_fd); + udsSendFd(fd, share_fd); + char data[PATH_MAX]; + while (true) { + printf("Input message to send: "); + gets(data); + if (strncmp(data,"quit",4) == 0) { + break; + } + ret = write(fd, data, strlen(data)); + if (ret < 0) { + printf("Fail to write,%s\n", strerror(errno)); + } + if(testfile == NULL && read(pipefd[0], buffer, sizeof(buffer))) { + printf("read from pipe: %s\n", buffer); + } + + } + close(fd); + return 0; +} \ No newline at end of file diff --git a/qtfs/test/scmright_test/makefile b/qtfs/test/scmright_test/makefile new file mode 100644 index 0000000..6a3cb87 --- /dev/null +++ b/qtfs/test/scmright_test/makefile @@ -0,0 +1,15 @@ +.PHONY: all + +all: libscm.so client server + +client: libscm.so client.c + gcc client.c -L. -lscm -o client + +server: libscm.so server.c + gcc server.c -L. -lscm -o server + +libscm.so: scm_rights.c + gcc -fPIC --share scm_rights.c -o libscm.so + +clean: + rm -f client server libscm.so *.o \ No newline at end of file diff --git a/qtfs/test/scmright_test/scm_rights.c b/qtfs/test/scmright_test/scm_rights.c new file mode 100644 index 0000000..19eeea7 --- /dev/null +++ b/qtfs/test/scmright_test/scm_rights.c @@ -0,0 +1,96 @@ +#include +# include +# include +# include +# include +# include +# include +# include +# include +# include +#include +#include +int +udsRecvFd(int sock, int fdflags) +{ + int byte = 0; + struct iovec iov; + struct msghdr msg; + int fd = -1; + ssize_t len; + struct cmsghdr *cmsg; + char buf[CMSG_SPACE(sizeof(fd))]; + int fdflags_recvmsg = fdflags & O_CLOEXEC ? MSG_CMSG_CLOEXEC : 0; + + if ((fdflags & ~O_CLOEXEC) != 0) { + errno = EINVAL; + return -1; + } + + /* send at least one char */ + memset(&msg, 0, sizeof(msg)); + iov.iov_base = &byte; + iov.iov_len = 4; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_name = NULL; + msg.msg_namelen = 0; + + msg.msg_control = buf; + msg.msg_controllen = sizeof(buf); + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_len = CMSG_LEN(sizeof(fd)); + /* Initialize the payload: */ + memcpy(CMSG_DATA(cmsg), &fd, sizeof(fd)); + msg.msg_controllen = cmsg->cmsg_len; + + len = recvmsg(sock, &msg, fdflags_recvmsg); + if (len < 0) + return -1; + + cmsg = CMSG_FIRSTHDR(&msg); + printf("recv msg type:%d SCM_RIGHTS:%d, byte:%d\n", cmsg->cmsg_type, SCM_RIGHTS, byte); + /* be paranoiac */ + if (len == 0 || cmsg == NULL || cmsg->cmsg_len != CMSG_LEN(sizeof(fd)) + || cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) { + /* fake errno: at end the file is not available */ + errno = len ? EACCES : ENOTCONN; + return -1; + } + + memcpy(&fd, CMSG_DATA(cmsg), sizeof(fd)); + + return fd; +} +int udsSendFd(int sock, int fd) +{ + char byte = 0; + struct iovec iov; + struct msghdr msg; + struct cmsghdr *cmsg; + char buf[CMSG_SPACE(sizeof(fd))]; + + /* send at least one char */ + memset(&msg, 0, sizeof(msg)); + iov.iov_base = &byte; + iov.iov_len = 1; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_name = NULL; + msg.msg_namelen = 0; + + msg.msg_control = buf; + msg.msg_controllen = sizeof(buf); + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(fd)); + /* Initialize the payload: */ + memcpy(CMSG_DATA(cmsg), &fd, sizeof(fd)); + msg.msg_controllen = cmsg->cmsg_len; + + if (sendmsg(sock, &msg, 0) != iov.iov_len) + return -1; + return 0; +} \ No newline at end of file diff --git a/qtfs/test/scmright_test/server.c b/qtfs/test/scmright_test/server.c new file mode 100644 index 0000000..62c1e4f --- /dev/null +++ b/qtfs/test/scmright_test/server.c @@ -0,0 +1,88 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "uds.h" + +static const char *HEAD = "write from server:\n"; + +int main(int argc, char **argv) +{ + char *path = NULL; + if (argc != 2) { + printf("Usage: %s pathtosock\n", argv[0]); + return -EINVAL; + } + path=argv[1]; + int fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) { + printf("Fail to create socket: %s\n", strerror(errno)); + return -1; + } + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + memcpy(addr.sun_path, path, strlen(path) + 1); + socklen_t size = offsetof(struct sockaddr_un, sun_path) + strlen(path) + 1;; + unlink(path); + printf("begin bind %s\n", addr.sun_path); + int ret = bind(fd, (struct sockaddr*)&addr, sizeof(addr)); + if (ret < 0) { + printf("Fail to bind socket: %s\n", strerror(errno)); + return ret; + } + printf("begin listen...\n"); + ret = listen(fd, 10); + if (ret < 0) { + printf("Fail to listen, %s\n", strerror(errno)); + close(fd); + return ret; + } + size = sizeof(addr); + while (1) { + printf("try accept %s...\n", addr.sun_path); + int new_fd = accept(fd, (struct sockaddr*)&addr, &size); + if (new_fd < 0) { + printf("Fail to accpet, %s\n", strerror(errno)); + close(fd); + return -1; + } + printf("try udsRecvFd...\n"); + int share_fd = udsRecvFd(new_fd, 0); + if (share_fd) { + printf("server write to share_fd[%d]\n", share_fd); + write(share_fd, HEAD, strlen(HEAD)); + } + printf("receive share_fd=%d\n", share_fd); + char buf[PATH_MAX]; + while (1) { + printf("try read from uds client...\n"); + ret = read(new_fd, buf, PATH_MAX); + if (ret <= 0) { + printf("closed: %s\n", strerror(errno)); + close(fd); + close(new_fd); + unlink(path); + return -1; + } + buf[ret] = '\0'; + if (strncmp(buf, "quit", 4) == 0) { + printf("receive quit\n"); + break; + } + printf("readmsg from client: %s\n", buf); + write(share_fd, buf, ret); + } + close(share_fd); + close(new_fd); + } + close(fd); +} \ No newline at end of file diff --git a/qtfs/test/scmright_test/uds.h b/qtfs/test/scmright_test/uds.h new file mode 100644 index 0000000..be47d06 --- /dev/null +++ b/qtfs/test/scmright_test/uds.h @@ -0,0 +1,9 @@ +#ifndef __UDS_H__ +#define __UDS_H__ + + +int udsRecvFd(int sock, int fdflags); + +int udsSendFd(int sock, int fd); + +#endif \ No newline at end of file diff --git a/qtfs/utils/License b/qtfs/utils/License new file mode 100644 index 0000000..3912109 --- /dev/null +++ b/qtfs/utils/License @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/qtfs/utils/qtfs_utils.h b/qtfs/utils/qtfs_utils.h new file mode 100644 index 0000000..4717771 --- /dev/null +++ b/qtfs/utils/qtfs_utils.h @@ -0,0 +1,55 @@ +/****************************************************************************** + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * qtfs licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + * Author: Liqiang + * Create: 2023-03-20 + * Description: + *******************************************************************************/ + +#ifndef __QTFS_UTILS_H__ +#define __QTFS_UTILS_H__ + +#define QTFS_UTILS_DEV "/dev/qtfs_utils" + +// QTFS provide some remote capability +#define QTUTIL_IOCTL_CAPA_MAGIC 'U' +enum { + _QTUTIL_IOCTL_CAPA_PORT_INUSE, +}; +#define QTUTIL_CAPA(CAP) _IO(QTUTIL_IOCTL_CAPA_MAGIC, CAP) +#define QTUTIL_CAPA_PORT_INUSE QTUTIL_CAPA(_QTUTIL_IOCTL_CAPA_PORT_INUSE) + +// QTFS provide some remote syscalls +#define QTUTIL_IOCTL_SC_MAGIC 'S' +enum { + _QTUTIL_IOCTL_SYSCALL_KILL, + _QTUTIL_IOCTL_SYSCALL_SCHED_SETAFFINITY, + _QTUTIL_IOCTL_SYSCALL_SCHED_GETAFFINITY, +}; +#define QTUTIL_SYSCALL(SC) _IO(QTUTIL_IOCTL_SC_MAGIC, SC) +#define QTUTIL_SC_KILL QTUTIL_SYSCALL(_QTUTIL_IOCTL_SYSCALL_KILL) +#define QTUTIL_SC_SCHED_SETAFFINITY QTUTIL_SYSCALL(_QTUTIL_IOCTL_SYSCALL_SCHED_SETAFFINITY) +#define QTUTIL_SC_SCHED_GETAFFINITY QTUTIL_SYSCALL(_QTUTIL_IOCTL_SYSCALL_SCHED_GETAFFINITY) + +struct qtsc_kill { + int pid; + int signum; +}; + +// sched getaffinity and set affinity +struct qtsc_sched_affinity { + int pid; + unsigned int len; + unsigned long *user_mask_ptr; +}; + + +#endif + diff --git a/qtfs/utils/utils.c b/qtfs/utils/utils.c new file mode 100644 index 0000000..07bac2a --- /dev/null +++ b/qtfs/utils/utils.c @@ -0,0 +1,202 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "conn.h" +#include "log.h" +#include "qtfs_utils.h" +#include "req.h" +#include "qtfs-mod.h" + +#ifndef QTFS_CLIENT +#error "QTFS utils only use in qtfs client." +#endif + +long qtfs_utils_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +long qtfs_capability_ioctl(struct qtfs_conn_var_s *pvar, struct file *file, unsigned int cmd, unsigned long arg); +long qtfs_syscall_ioctl(struct qtfs_conn_var_s *pvar, struct file *file, unsigned int cmd, unsigned long arg); + +struct file_operations qtfs_utils_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = qtfs_utils_ioctl, +}; + +static struct miscdevice qtfs_utils_dev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "qtfs_utils", + .fops = &qtfs_utils_fops, +}; + +int qtfs_utils_register(void) +{ + int ret = misc_register(&qtfs_utils_dev); + + if (ret) { + qtfs_err("qtfs utils dev register failed, ret:%d", ret); + return -EFAULT; + } + return 0; +} + +void qtfs_utils_destroy(void) +{ + misc_deregister(&qtfs_utils_dev); + + return; +} + +long qtfs_utils_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + long ret = -EINVAL; + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + + if (pvar == NULL) { + qtfs_err("utils ioctl get pvar failed, cmd type:%c nr:%d", _IOC_TYPE(cmd), _IOC_NR(cmd)); + return -EFAULT; + } + switch (_IOC_TYPE(cmd)) { + case QTUTIL_IOCTL_CAPA_MAGIC: + ret = qtfs_capability_ioctl(pvar, file, cmd, arg); + qtfs_info("utils capability ioctl nr:%d", _IOC_NR(cmd)); + break; + + case QTUTIL_IOCTL_SC_MAGIC: + ret = qtfs_syscall_ioctl(pvar, file, cmd, arg); + qtfs_info("utils syscall ioctl nr:%d", _IOC_NR(cmd)); + break; + + default: + qtfs_err("Unsupport magic type:%c", _IOC_TYPE(cmd)); + break; + } + qtfs_conn_put_param(pvar); + return ret; +} + +long qtfs_capability_ioctl(struct qtfs_conn_var_s *pvar, struct file *file, unsigned int cmd, unsigned long arg) +{ + long ret = -ENOTSUPP; + return ret; +} + +static long qtfs_sc_kill(struct qtfs_conn_var_s *pvar, unsigned long arg) +{ + struct qtreq_sc_kill *req; + struct qtrsp_sc_kill *rsp; + struct qtsc_kill karg; + + if (copy_from_user(&karg, (void *)arg, sizeof(struct qtsc_kill))) { + qtfs_err("copy args failed."); + return -EINVAL; + } + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + req->pid = karg.pid; + req->signum = karg.signum; + rsp = qtfs_remote_run(pvar, QTFS_SC_KILL, sizeof(struct qtreq_sc_kill)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_err("qtfs remote kill faile."); + return -EFAULT; + } + qtfs_info("qtfs remote kill pid:%d sig:%d success:%ld", req->pid, req->signum, rsp->ret); + return rsp->ret; +} + +static long qtfs_sc_setaffinity(struct qtfs_conn_var_s *pvar, unsigned long arg) +{ + struct qtreq_sc_sched_affinity *req; + struct qtrsp_sc_sched_affinity *rsp; + struct qtsc_sched_affinity karg; + + if (copy_from_user(&karg, (void *)arg, sizeof(struct qtsc_sched_affinity))) { + qtfs_err("copy args failed."); + return -EINVAL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + req->type = SC_SET; + req->pid = karg.pid; + req->len = (karg.len > AFFINITY_MAX_LEN) ? AFFINITY_MAX_LEN : karg.len; + if (copy_from_user(req->user_mask_ptr, karg.user_mask_ptr, req->len)) { + qtfs_err("copy from user mask ptr failed, len:%u", karg.len); + return -EFAULT; + } + rsp = qtfs_remote_run(pvar, QTFS_SC_SCHED_SETAFFINITY, sizeof(struct qtreq_sc_sched_affinity) + karg.len * sizeof(unsigned long)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_err("qtfs remote set affinit failed, pid:%d len:%u", karg.pid, karg.len); + return -EFAULT; + } + qtfs_info("qtfs remote set affinity successed pid:%d len:%u return:%ld", karg.pid, karg.len, rsp->ret); + return rsp->ret; +} + +static long qtfs_sc_getaffinity(struct qtfs_conn_var_s *pvar, unsigned long arg) +{ + struct qtreq_sc_sched_affinity *req; + struct qtrsp_sc_sched_affinity *rsp; + struct qtsc_sched_affinity karg; + + if (copy_from_user(&karg, (void *)arg, sizeof(struct qtsc_sched_affinity))) { + qtfs_err("copy args failed."); + return -EINVAL; + } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + req->type = SC_GET; + req->pid = karg.pid; + req->len = (karg.len > AFFINITY_MAX_LEN) ? AFFINITY_MAX_LEN : karg.len; + rsp = qtfs_remote_run(pvar, QTFS_SC_SCHED_GETAFFINITY, sizeof(struct qtreq_sc_sched_affinity)); + if (IS_ERR_OR_NULL(rsp)) { + qtfs_err("qtfs remote get affinit failed, pid:%d len:%u", karg.pid, karg.len); + return -EFAULT; + } + // len == 0 means failed + if (rsp->len <= 0 || rsp->len > req->len || + copy_to_user(karg.user_mask_ptr, rsp->user_mask_ptr, rsp->len)) { + qtfs_err("copy user mask ptr failed rsp len:%u valid len:%u", rsp->len, karg.len); + return -EINVAL; + } + return rsp->ret; +} + +long qtfs_syscall_ioctl(struct qtfs_conn_var_s *pvar, struct file *file, unsigned int cmd, unsigned long arg) +{ + long ret = -EINVAL; + + switch (cmd) { + case QTUTIL_SC_KILL: + ret = qtfs_sc_kill(pvar, arg); + break; + + case QTUTIL_SC_SCHED_SETAFFINITY: + ret = qtfs_sc_setaffinity(pvar, arg); + break; + + case QTUTIL_SC_SCHED_GETAFFINITY: + ret = qtfs_sc_getaffinity(pvar, arg); + break; + + default: + qtfs_err("syscall ioctl not support magic:%c number:%d", _IOC_TYPE(cmd), _IOC_NR(cmd)); + break; + } + return ret; +} + diff --git a/script.md b/script.md new file mode 100644 index 0000000..313adc0 --- /dev/null +++ b/script.md @@ -0,0 +1,66 @@ +### 第一幕 介绍环境信息 +1> 大禹/Host硬件信息。 + -- 略 + +2> 操作系统信息。 + +-- 命令1，HOST/DPU: cat /etc/os-release + 说明：查看OS发行版信息。 + +3> 仅dpu侧有libvirt，host上无关键进程。 +（逐步标出关键信息） + 第3步命令列表： + 命令1，DPU/HOST: ps -ef |grep libvirtd + 说明：在DPU上查询到libvirtd守护进程存在，而HOST上没有libvirt守护进程。 + + 命令2，DPU/HOST: ps -ef |grep qemu + 说明：在DPU和HOST上初始状态均没有虚拟机相关进程存在。 + +### 第二幕 DPU侧 +DPU> 1> virsh define / start （字幕：创建虚拟机）操作 + 第1步命令： + 命令1，DPU: virsh define /root/remote_vm.xml + 说明：从xml文件创建虚机配置。 + + 命令2，DPU: virsh start remote_vm + 说明：启动刚刚创建的虚机remote_vm + +DPU> 1.1 > virsh list（字幕：虚拟机已创建）结果 + 第2步命令： + 命令1，DPU: virsh list + 说明：查询到libvirt管理的虚机，能查询到remote_vm的状态为running + + +DPU> 2> ps查询进程状态，无qemu（但有proxy进程组件）结果 + 第3步命令： + 命令1，DPU: ps -ef |grep qemu + 说明：能查询到一个/usr/bin/rexec进程带有qemu关键字的参数，这是一个虚机在DPU的代理进程，不是真正的虚机。 + + + +DPU> 4> virsh console查看虚拟机 （字幕：虚拟机可访问）结果 + 第4步命令： + 命令1，DPU：virsh console remote_vm + 说明：账号输入“root”，密码输入“openEuler12#$”登入虚拟机。 + + 命令2，在刚刚登入的虚机内进行一些简单的操作： + cat /proc/cpuinfo |grep processor (查看虚机CPU核数) + cat /proc/cpuinfo |grep QEMU (查看CPU MODEL信息) + cat /proc/meminfo |grep MemTotal (查看虚机内存总量) + + 命令3，在登入的虚机中退出虚机控制台：ctrl + ] + + +### 第三幕 HOST侧， +HOST> 3> ps查询进程状态，有qemu 结果 + 命令1，HOST: ps -ef |grep qemu + 说明：能查询到一个qemu-kvm真实虚机进程，查看其参数中有-name guest=remote_vm字样，说明真正的虚机运行在host。 + +第四幕 DPU侧删除虚拟机 +DPU> virsh destroy + 命令，DPU：virsh destroy remote_vm + 说明：关闭虚机。 + +DPU> virsh list + 命令，DPU：virsh list + 说明：没有running状态虚机显示，如果输入virsh list --all则显示remote_vm虚机为shut off状态 + diff --git a/virsh_demo.xml b/virsh_demo.xml new file mode 100644 index 0000000..375749a --- /dev/null +++ b/virsh_demo.xml @@ -0,0 +1,110 @@ + + + + lq + 7c0189f5-457c-1834-9387-73de2860b088 + 4194304 + 4194304 + + 4194304 + + 4 + 3 + + + + + + + hvm + /usr/share/edk2/aarch64/QEMU_EFI-pflash.raw + /var/lib/libvirt/qemu/nvram/common_VARS.fd + + + + + + + + + destroy + restart + destroy + + /usr/bin/qemu-rexec + + + + +
+ + + + + +
+ + +
+ + +
+ + + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + + +
+ + + + + + + + + + + +
+ + + + +