diff --git a/centos_iso_image.inc b/centos_iso_image.inc index 301a5c3c..977589b6 100644 --- a/centos_iso_image.inc +++ b/centos_iso_image.inc @@ -18,3 +18,7 @@ nfv-plugins nfv-tools nfv-vim nfv-client + +# mtce-guest +mtce-guestAgent +mtce-guestServer diff --git a/centos_pkg_dirs b/centos_pkg_dirs index 081f5e8a..2ad8b64e 100644 --- a/centos_pkg_dirs +++ b/centos_pkg_dirs @@ -3,3 +3,4 @@ guest-client guest-agent guest-comm nfv +mtce-guest diff --git a/mtce-guest/PKG-INFO b/mtce-guest/PKG-INFO new file mode 100644 index 00000000..3bbe0d53 --- /dev/null +++ b/mtce-guest/PKG-INFO @@ -0,0 +1,13 @@ +Metadata-Version: 1.1 +Name: mtce-guest +Version: 1.0 +Summary: Maintenance Guest Server/Agent Package +Home-page: +Author: Windriver +Author-email: info@windriver.com +License: Apache-2.0 + +Description: Maintenance Guest Server/Agent package containing daemons, resource and + file system daemons, as well as support files for each. + +Platform: UNKNOWN diff --git a/mtce-guest/centos/build_srpm.data b/mtce-guest/centos/build_srpm.data new file mode 100644 index 00000000..f8556950 --- /dev/null +++ b/mtce-guest/centos/build_srpm.data @@ -0,0 +1,3 @@ +SRC_DIR="src" +TIS_PATCH_VER=139 +BUILD_IS_SLOW=5 diff --git a/mtce-guest/centos/mtce-guest.spec b/mtce-guest/centos/mtce-guest.spec new file mode 100644 index 00000000..ea447bbc --- /dev/null +++ b/mtce-guest/centos/mtce-guest.spec @@ -0,0 +1,222 @@ +Summary: Maintenance Guest Server/Agent Package +Name: mtce-guest +Version: 1.0 +%define patchlevel %{tis_patch_ver} +Release: %{tis_patch_ver}%{?_tis_dist} + +License: Apache-2.0 +Group: base +Packager: Wind River +URL: unknown + +Source0: %{name}-%{version}.tar.gz + +BuildRequires: openssl +BuildRequires: openssl-devel +BuildRequires: json-c +BuildRequires: json-c-devel +BuildRequires: libevent +BuildRequires: libevent-devel +BuildRequires: libuuid +BuildRequires: libuuid-devel +BuildRequires: fm-common +BuildRequires: fm-common-dev +BuildRequires: guest-client-devel +BuildRequires: mtce-common-dev >= 1.0 +BuildRequires: systemd-devel +BuildRequires: cppcheck + +%description +Maintenance Guest Agent Service and Server assists in VM guest +heartbeat control and failure reporting at the controller level. + +%package -n mtce-guestAgent +Summary: Maintenance Guest Agent Package +Group: base +Requires: dpkg +Requires: time +Requires: libjson-c.so.2()(64bit) +Requires: libstdc++.so.6(CXXABI_1.3)(64bit) +Requires: librt.so.1(GLIBC_2.2.5)(64bit) +Requires: libfmcommon.so.1()(64bit) +Requires: libstdc++.so.6(GLIBCXX_3.4.9)(64bit) +Requires: fm-common >= 1.0 +Requires: libc.so.6(GLIBC_2.2.5)(64bit) +Requires: libstdc++.so.6(GLIBCXX_3.4.11)(64bit) +Requires: /bin/sh +Requires: librt.so.1()(64bit) +Requires: libc.so.6(GLIBC_2.3)(64bit) +Requires: libc.so.6(GLIBC_2.14)(64bit) +Requires: libpthread.so.0(GLIBC_2.2.5)(64bit) +Requires: librt.so.1(GLIBC_2.3.3)(64bit) +Requires: libgcc_s.so.1(GCC_3.0)(64bit) +Requires: libevent >= 2.0.21 +Requires: libevent-2.0.so.5()(64bit) +Requires: libuuid.so.1()(64bit) +Requires: libm.so.6()(64bit) +Requires: rtld(GNU_HASH) +Requires: libstdc++.so.6()(64bit) +Requires: libc.so.6()(64bit) +Requires: libgcc_s.so.1()(64bit) +Requires: libstdc++.so.6(GLIBCXX_3.4)(64bit) +Requires: libstdc++.so.6(GLIBCXX_3.4.15)(64bit) +Requires: libpthread.so.0()(64bit) + + +%description -n mtce-guestAgent +Maintenance Guest Agent Service assists in +VM guest heartbeat control and failure reporting at the controller +level. + +%package -n mtce-guestServer +Summary: Maintenance Guest Server Package +Group: base +Requires: util-linux +Requires: /bin/bash +Requires: /bin/systemctl +Requires: dpkg +Requires: libjson-c.so.2()(64bit) +Requires: libstdc++.so.6(CXXABI_1.3)(64bit) +Requires: librt.so.1(GLIBC_2.2.5)(64bit) +Requires: libfmcommon.so.1()(64bit) +Requires: libstdc++.so.6(GLIBCXX_3.4.9)(64bit) +Requires: fm-common >= 1.0 +Requires: libc.so.6(GLIBC_2.2.5)(64bit) +Requires: libstdc++.so.6(GLIBCXX_3.4.11)(64bit) +Requires: /bin/sh +Requires: librt.so.1()(64bit) +Requires: libc.so.6(GLIBC_2.3)(64bit) +Requires: libc.so.6(GLIBC_2.14)(64bit) +Requires: libpthread.so.0(GLIBC_2.2.5)(64bit) +Requires: librt.so.1(GLIBC_2.3.3)(64bit) +Requires: libgcc_s.so.1(GCC_3.0)(64bit) +Requires: libevent >= 2.0.21 +Requires: libevent-2.0.so.5()(64bit) +Requires: libuuid.so.1()(64bit) +Requires: libm.so.6()(64bit) +Requires: rtld(GNU_HASH) +Requires: libstdc++.so.6()(64bit) +Requires: libc.so.6(GLIBC_2.4)(64bit) +Requires: libc.so.6()(64bit) +Requires: libgcc_s.so.1()(64bit) +Requires: libstdc++.so.6(GLIBCXX_3.4)(64bit) +Requires: libstdc++.so.6(GLIBCXX_3.4.15)(64bit) +Requires: libpthread.so.0()(64bit) + +%description -n mtce-guestServer +Maintenance Guest Server assists in VM guest +heartbeat control and failure reporting at the compute level. + +%define local_dir /usr/local +%define local_bindir %{local_dir}/bin +%define local_sbindir %{local_dir}/sbin +%define local_etc_pmond %{_sysconfdir}/pmon.d +%define local_etc_servicesd %{_sysconfdir}/services.d +%define local_etc_logrotated %{_sysconfdir}/logrotate.d +%define ocf_resourced /usr/lib/ocf/resource.d + +%prep +%setup + +# build mtce-guestAgent and mtce-guestServer package +%build +VER=%{version} +MAJOR=$(echo $VER | awk -F . '{print $1}') +MINOR=$(echo $VER | awk -F . '{print $2}') +make MAJOR=$MAJOR MINOR=$MINOR %{?_smp_mflags} build + +%global _buildsubdir %{_builddir}/%{name}-%{version} + +# install mtce-guestAgent and mtce-guestServer package +%install +VER=%{version} +MAJOR=$(echo $VER | awk -F . '{print $1}') +MINOR=$(echo $VER | awk -F . '{print $2}') + +install -m 755 -d %{buildroot}%{_sysconfdir} +install -m 755 -d %{buildroot}/usr +install -m 755 -d %{buildroot}/%{_bindir} +install -m 755 -d %{buildroot}/usr/local +install -m 755 -d %{buildroot}%{local_bindir} +install -m 755 -d %{buildroot}/usr/local/sbin +install -m 755 -d %{buildroot}/%{_sbindir} +install -m 755 -d %{buildroot}/lib +install -m 755 -d %{buildroot}%{_sysconfdir}/mtc +install -m 755 -d %{buildroot}%{_sysconfdir}/mtc/tmp + +# resource agent stuff +install -m 755 -d %{buildroot}/usr/lib +install -m 755 -d %{buildroot}/usr/lib/ocf +install -m 755 -d %{buildroot}/usr/lib/ocf/resource.d +install -m 755 -d %{buildroot}/usr/lib/ocf/resource.d/platform +install -m 755 -p -D %{_buildsubdir}/scripts/guestAgent.ocf %{buildroot}/usr/lib/ocf/resource.d/platform/guestAgent + +# config files +install -m 644 -p -D %{_buildsubdir}/scripts/guest.ini %{buildroot}%{_sysconfdir}/mtc/guestAgent.ini +install -m 644 -p -D %{_buildsubdir}/scripts/guest.ini %{buildroot}%{_sysconfdir}/mtc/guestServer.ini + +# binaries +install -m 755 -p -D %{_buildsubdir}/guestServer %{buildroot}/%{local_bindir}/guestServer +install -m 755 -p -D %{_buildsubdir}/guestAgent %{buildroot}/%{local_bindir}/guestAgent + +# init script files +install -m 755 -p -D %{_buildsubdir}/scripts/guestServer %{buildroot}%{_sysconfdir}/init.d/guestServer +install -m 755 -p -D %{_buildsubdir}/scripts/guestAgent %{buildroot}%{_sysconfdir}/init.d/guestAgent + +# systemd service files +install -m 644 -p -D %{_buildsubdir}/scripts/guestServer.service %{buildroot}%{_unitdir}/guestServer.service +install -m 644 -p -D %{_buildsubdir}/scripts/guestAgent.service %{buildroot}%{_unitdir}/guestAgent.service + +# process monitor config files +install -m 755 -d %{buildroot}%{local_etc_pmond} +install -m 644 -p -D %{_buildsubdir}/scripts/guestServer.pmon %{buildroot}%{local_etc_pmond}/guestServer.conf + +# log rotation +install -m 755 -d %{buildroot}%{_sysconfdir}/logrotate.d +install -m 644 -p -D %{_buildsubdir}/scripts/guestAgent.logrotate %{buildroot}%{local_etc_logrotated}/guestAgent.logrotate +install -m 644 -p -D %{_buildsubdir}/scripts/guestServer.logrotate %{buildroot}%{local_etc_logrotated}/guestServer.logrotate + +# volatile directores +install -m 755 -d %{buildroot}/var +install -m 755 -d %{buildroot}/var/run + +# enable all services in systemd +%post -n mtce-guestServer +/bin/systemctl enable guestServer.service + +%files -n mtce-guestAgent +%license LICENSE +%defattr(-,root,root,-) + +# create mtc and its tmp dir +%dir %{_sysconfdir}/mtc +%dir %{_sysconfdir}/mtc/tmp + +# config files - non-modifiable +%{_sysconfdir}/mtc/guestAgent.ini + +%{_unitdir}/guestAgent.service +%{local_etc_logrotated}/guestAgent.logrotate +%{ocf_resourced}/platform/guestAgent + +%{_sysconfdir}/init.d/guestAgent +%{local_bindir}/guestAgent + +%files -n mtce-guestServer +%license LICENSE +%defattr(-,root,root,-) + +# create mtc and its tmp dir +%dir %{_sysconfdir}/mtc +%dir %{_sysconfdir}/mtc/tmp + +# config files - non-modifiable +%{_sysconfdir}/mtc/guestServer.ini + +%{local_etc_pmond}/guestServer.conf +%{local_etc_logrotated}/guestServer.logrotate +%{_unitdir}/guestServer.service + +%{_sysconfdir}/init.d/guestServer +%{local_bindir}/guestServer + diff --git a/mtce-guest/src/LICENSE b/mtce-guest/src/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/mtce-guest/src/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/mtce-guest/src/Makefile b/mtce-guest/src/Makefile new file mode 100644 index 00000000..58a608b6 --- /dev/null +++ b/mtce-guest/src/Makefile @@ -0,0 +1,36 @@ +# +# Copyright (c) 2015-2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +SRCS = guestClass.cpp guestInstClass.cpp \ + guestSvrFsm.cpp guestSvrHdlr.cpp \ + guestServer.cpp guestAgent.cpp \ + guestHttpSvr.cpp guestHttpUtil.cpp guestVimApi.cpp \ + guestUtil.cpp guestSvrUtil.cpp guestSvrMsg.cpp \ + guestVirtio.cpp guestStubs.cpp +AGENT_OBJS = \ + guestAgent.o guestClass.o guestHttpSvr.o guestHttpUtil.o guestVimApi.o guestUtil.o guestStubs.o +SERVER_OBJS = \ + guestServer.o guestInstClass.o \ + guestSvrFsm.o guestSvrHdlr.o \ + guestSvrMsg.o guestVirtio.o \ + guestUtil.o guestSvrUtil.o + +OBJS = $(SRCS:.cpp=.o) + +CCPFLAGS = -g -O2 -Wall -Wextra -Werror +LDLIBS = -lstdc++ -ldaemon -lcommon -lfmcommon -ljson-c -levent -lrt -lcrypto -luuid +INCLUDES = -I. -I/usr/include/mtce-common -I/usr/include/mtce-daemon + +build: $(OBJS) + $(CXX) $(CCPFLAGS) $(AGENT_OBJS) $(LDLIBS) -L. -o guestAgent + $(CXX) $(CCPFLAGS) $(SERVER_OBJS) $(LDLIBS) -L. -o guestServer + +.cpp.o: + $(CXX) $(INCLUDES) $(CCPFLAGS) $(EXTRACCFLAGS) -c $< -o $@ + +clean: + @rm -v -f $(OBJ) guestAgent guestServer *.o *.a + diff --git a/mtce-guest/src/README.txt b/mtce-guest/src/README.txt new file mode 100644 index 00000000..92cb11e9 --- /dev/null +++ b/mtce-guest/src/README.txt @@ -0,0 +1,67 @@ +This maintenance common guest service provides a means of guest heartbeat +orchestration under VIM (system management) control. + +1. Packaging and Initialization: + + The packaging of the heartbeat daemon and heartbeat_init + script are modified: + + a. image.inc and filter_out packaging files are modified to exclude the + heartbeat daemon from being packaged on the controller. + + b. because the heartbeat daemon is still packaged on the compute + heartbeat_init script is modified to prevent the heartbeat + daemon from being spawned on the compute host. + +2. Compute Function: Heartbeats the guest and reports failures. + + Package: cgts-mtce-common-guestServer-1.0-r54.0.x86_64.rpm + Binary: /usr/local/bin/guestServer + Init: /etc/init.d/guestServer + Managed: /etc/pmon.d/guestServer.pmon + +3. Controller Function: Guest heartbeat control and event proxy + + Package: cgts-mtce-common-guestAgent-x.x-rxx.x86_64.rpm + Binary: /usr/local/bin/guestAgent + Init: /usr/lib/ocf/resource.d/platform/guestAgent + Managed: by SM + + The heartbeat daemon that did run on the controller is replaced by a new + guestAgent daemon performing the following functions + + a. HTTP Command Receiver : to which the VIM sends instance control commands. + b. HTTP Event Transmitter: to which the daemon can send instance failure + events and state query commands to the VIM. + c. State query audit + +Behavioral Executive Summary: + +The guestServer daemon (on the compute) listens for (using inotify) 'uuid' +UNIX named heartbeat communication channels that nova:libvirt creates and +opens in /var/lib/libvirt/qemu whenever an instance is created. Example: + +/var/lib/libvirt/qemu/cgcs.heartbeat.02e172a9-aeae-4cef-a6bc-7eb9de7825d6.sock + +The guestServer connects (and auto reconnects) to these channels when they are +created or modified and disconnects from them when deleted. + +Once connected, the guestServer listens for TCP messages over that UNIX named +socket. + +If a guest supports heartbeating then it will run the heartbeat_init script +during its initialization process. Once the heartbeat daemon is running it +will periodically send init request messages to the libvirt pipe that, +on the host side, the guestServer is listening for. + +on receipt of an init message, the guestServer will extract name, timeout +and corrective action info from it and then send back an 'init_ack' followed +by a continuous heartbeat cycle consisting of sending a 'challenge' request +messages and expecting a correct computational responses within a guest specified +heartbeat window. Failure to comply will result in a corrective action that was +specified in the init message from the guest. + +The VIM is responsible for enabling and disabling heartbeat fault reporting as +well as taking the guest specified corrective action in he event of a heartbeat +failure. + diff --git a/mtce-guest/src/guestAgent.cpp b/mtce-guest/src/guestAgent.cpp new file mode 100644 index 00000000..c2bb5647 --- /dev/null +++ b/mtce-guest/src/guestAgent.cpp @@ -0,0 +1,1429 @@ +/* + * Copyright (c) 2013, 2016 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + + /** + * @file + * Wind River CGTS Platform Guest Services Agent Daemon + * + * Services: heartbeat + */ + +#include +#include +#include +#include +#include +#include +#include +#include /* for ... hostent */ +#include +#include +#include +#include +#include +#include /* for ... close and usleep */ +#include +#include /* for ... RTMGRP_LINK */ + +using namespace std; + +#include "daemon_ini.h" /* Ini Parser Header */ +#include "daemon_common.h" /* Common definitions and types for daemons */ +#include "daemon_option.h" /* Common options for daemons */ + +#include "nodeBase.h" +#include "nodeMacro.h" /* for ... CREATE_NONBLOCK_INET_UDP_RX_SOCKET */ +#include "nodeUtil.h" /* for ... get_ip_addresses */ +#include "nodeTimers.h" /* maintenance timer utilities start/stop */ +#include "jsonUtil.h" /* for ... jsonApi_get_key_value */ +#include "httpUtil.h" /* for ... */ + +#include "guestBase.h" /* Guest services Base Header File */ +#include "guestClass.h" /* */ +#include "guestUtil.h" /* for ... guestUtil_inst_init */ + +#include "guestHttpUtil.h" /* for ... guestHttpUtil_init */ +#include "guestHttpSvr.h" /* for ... guestHttpSvr_init/_fini/_look */ +#include "guestVimApi.h" /* for ... guestVimApi_getHostState */ + +/* Where to send events */ +string guestAgent_ip = "" ; + +/* Process Monitor Control Structure */ +static ctrl_type _ctrl ; + +/** This heartbeat service inventory is tracked by + * the same nodeLinkClass that maintenance uses. + * + */ +guestHostClass hostInv ; +guestHostClass * get_hostInv_ptr ( void ) +{ + return (&hostInv); +} + +/** Setup the pointer */ +int module_init ( void ) +{ + return (PASS); +} + +msgSock_type * get_mtclogd_sockPtr ( void ) +{ + return (&_ctrl.sock.mtclogd); +} + +void daemon_sigchld_hdlr ( void ) +{ + ; /* dlog("Received SIGCHLD ... no action\n"); */ +} + +/** + * Daemon Configuration Structure - The allocated struct + * @see daemon_common.h for daemon_config_type struct format. + */ +static daemon_config_type guest_config ; + +#ifdef __cplusplus +extern "C" { +#endif +daemon_config_type * daemon_get_cfg_ptr () { return &guest_config ; } +#ifdef __cplusplus +} +#endif + +/* Cleanup exit handler */ +void daemon_exit ( void ) +{ + daemon_dump_info (); + daemon_files_fini (); + + /* Close the event socket */ + if ( _ctrl.sock.server_rx_sock ) + delete (_ctrl.sock.server_rx_sock); + + if ( _ctrl.sock.server_tx_sock ) + delete (_ctrl.sock.server_tx_sock); + + if ( _ctrl.sock.agent_rx_local_sock ) + delete (_ctrl.sock.agent_rx_local_sock); + + if ( _ctrl.sock.agent_rx_float_sock ) + delete (_ctrl.sock.agent_rx_float_sock); + + if ( _ctrl.sock.agent_tx_sock ) + delete (_ctrl.sock.agent_tx_sock); + + guestHttpSvr_fini (); + guestHttpUtil_fini (); + + fflush (stdout); + fflush (stderr); + + exit (0); +} + +#define CONFIG_CHALLENGE_PERIOD (1) + +int _self_provision ( void ) +{ + int rc = PASS ; + int waiting_msg = false ; + + hostInv.hostBase.my_float_ip.clear(); + hostInv.hostBase.my_local_ip.clear(); + + for ( ;; ) + { + get_ip_addresses ( hostInv.hostBase.my_hostname, + hostInv.hostBase.my_local_ip , + hostInv.hostBase.my_float_ip ); + + if ( hostInv.hostBase.my_float_ip.empty() || hostInv.hostBase.my_local_ip.empty() ) + { + if ( waiting_msg == false ) + { + ilog ("Waiting on ip address config ...\n"); + waiting_msg = true ; + + /* Flush the init data */ + fflush (stdout); + fflush (stderr); + } + mtcWait_secs (3); + } + else + { + break ; + } + daemon_signal_hdlr (); + } + return (rc); +} + + +/** Control Config Mask */ +// #define CONFIG_MASK (CONFIG_CHALLENGE_PERIOD) +/** Client Config mask */ +#define CONFIG_MASK (CONFIG_CLIENT_RX_PORT |\ + CONFIG_AGENT_RX_PORT |\ + CONFIG_MTC_CMD_PORT |\ + CONFIG_VIM_CMD_RX_PORT |\ + CONFIG_VIM_EVENT_RX_PORT |\ + CONFIG_MTC_EVENT_PORT) + +/* Startup config read */ +static int _config_handler ( void * user, + const char * section, + const char * name, + const char * value) +{ + daemon_config_type* config_ptr = (daemon_config_type*)user; + + if (MATCH("agent", "rx_port")) + { + config_ptr->agent_rx_port = atoi(value); + config_ptr->mask |= CONFIG_AGENT_RX_PORT ; + } + else if (MATCH("agent", "vim_cmd_port")) + { + config_ptr->vim_cmd_port = atoi(value); + config_ptr->mask |= CONFIG_VIM_CMD_RX_PORT ; + } + else if (MATCH("client", "rx_port")) + { + config_ptr->client_rx_port = atoi(value); + config_ptr->mask |= CONFIG_CLIENT_RX_PORT ; + } + else + { + return (PASS); + } + return (FAIL); +} + +/* Startup config read */ +static int mtc_config_handler ( void * user, + const char * section, + const char * name, + const char * value) +{ + daemon_config_type* config_ptr = (daemon_config_type*)user; + + if (MATCH("agent", "hbs_to_mtc_event_port")) + { + config_ptr->hbs_to_mtc_event_port = atoi(value); + config_ptr->mask |= CONFIG_MTC_EVENT_PORT ; + } + if (MATCH("agent", "mtc_to_guest_cmd_port")) + { + config_ptr->mtc_to_guest_cmd_port = atoi(value); + config_ptr->mask |= CONFIG_MTC_CMD_PORT ; + } + else + { + return (PASS); + } + return (FAIL); +} + +int _nfvi_handler( void * user, + const char * section, + const char * name, + const char * value) +{ + daemon_config_type* config_ptr = (daemon_config_type*)user; + + if (MATCH("guest-rest-api", "port")) + { + config_ptr->vim_event_port = atoi(value); + config_ptr->mask |= CONFIG_VIM_EVENT_RX_PORT ; + } + else + { + return (PASS); + } + return (FAIL); +} + +/* Read the mtc.ini settings into the daemon configuration */ +int daemon_configure ( void ) +{ + /* Read the ini */ + char config_fn[100] ; + guest_config.mask = 0 ; + sprintf ( &config_fn[0], "/etc/mtc/%s.ini", program_invocation_short_name ); + if (ini_parse(config_fn, _config_handler, &guest_config) < 0) + { + elog("Can't load '%s'\n", config_fn ); + return (FAIL_LOAD_INI); + } + + get_debug_options ( config_fn, &guest_config ); + + if (ini_parse(MTCE_CONF_FILE, mtc_config_handler, &guest_config) < 0) + { + elog("Can't load '%s'\n", MTCE_CONF_FILE ); + return (FAIL_LOAD_INI); + } + + if (ini_parse(NFVI_PLUGIN_CFG_FILE, _nfvi_handler, &guest_config) < 0) + { + elog ("Can't load '%s'\n", NFVI_PLUGIN_CFG_FILE ); + return (FAIL_LOAD_INI); + } + + /* Verify loaded config against an expected mask + * as an ini file fault detection method */ + if ( guest_config.mask != CONFIG_MASK ) + { + elog ("Error: Agent configuration failed (%x) (%x:%x)\n", + ((-1 ^ guest_config.mask) & CONFIG_MASK), + guest_config.mask, CONFIG_MASK ); + return (FAIL_INI_CONFIG); + } + + guest_config.mgmnt_iface = daemon_get_iface_master ( guest_config.mgmnt_iface ); + ilog("Interface : %s\n", guest_config.mgmnt_iface ); + ilog("Command Port: %d (rx) from mtcAgent\n", guest_config.mtc_to_guest_cmd_port ); + ilog("Event Port: %d (tx) to mtcAgent\n", guest_config.hbs_to_mtc_event_port ); + ilog("Command Port: %d (tx) to guestServer\n", guest_config.client_rx_port); + ilog("Event Port: %d (rx) from guestServer\n",guest_config.agent_rx_port ); + ilog("Command Port: %d (rx) from vim\n",guest_config.vim_cmd_port ); + ilog("Event Port: %d (rx) to vim\n",guest_config.vim_event_port ); + + /* provision this controller */ + if ( _self_provision () != PASS ) + { + elog ("Failed to self provision active controller\n"); + daemon_exit (); + } + + return (PASS); +} + +void _timer_handler ( int sig, siginfo_t *si, void *uc) +{ + timer_t * tid_ptr = (void**)si->si_value.sival_ptr ; + + /* Avoid compiler errors/warnings for parms we must + * have but currently do nothing with */ + sig=sig ; uc = uc ; + + if ( !(*tid_ptr) ) + { + // tlog ("Called with a NULL Timer ID\n"); + return ; + } + /* is base ctrl mtc timer */ + else if (( *tid_ptr == _ctrl.timer.tid ) ) + { + mtcTimer_stop_int_safe ( _ctrl.timer ); + _ctrl.timer.ring = true ; + } + + /* is base object mtc timer */ + else if (( *tid_ptr == hostInv.audit_timer.tid ) ) + { + mtcTimer_stop_int_safe ( hostInv.audit_timer ); + hostInv.audit_timer.ring = true ; + } + else + { + mtcTimer_stop_tid_int_safe (tid_ptr); + } +} + +int mtclogd_port_init ( ctrl_type * ctrl_ptr ) +{ + int rc = PASS ; + int port = ctrl_ptr->sock.mtclogd.port = daemon_get_cfg_ptr()->daemon_log_port ; + CREATE_REUSABLE_INET_UDP_TX_SOCKET ( LOOPBACK_IP, + port, + ctrl_ptr->sock.mtclogd.sock, + ctrl_ptr->sock.mtclogd.addr, + ctrl_ptr->sock.mtclogd.port, + ctrl_ptr->sock.mtclogd.len, + "mtc logger message", + rc ); + if ( rc ) + { + elog ("Failed to setup messaging to mtclogd on port %d\n", port ); + } + return (rc); +} + +/****************************/ +/* Initialization Utilities */ +/****************************/ + +/* Construct the messaging sockets * + * 1. multicast transmit socket * + * 2. unicast receive socket */ +int _socket_init ( void ) +{ + int rc = PASS ; + + guestAgent_ip = getipbyname ( CONTROLLER ); + ilog ("ControllerIP: %s\n", guestAgent_ip.c_str()); + + /* Read the ports the socket struct */ + _ctrl.sock.agent_rx_port = guest_config.agent_rx_port ; + _ctrl.sock.server_rx_port = guest_config.client_rx_port ; + + /******************************************************************/ + /* UDP Tx Message Socket Towards guestServer */ + /******************************************************************/ + + _ctrl.sock.agent_tx_sock = new msgClassTx(guestAgent_ip.c_str(), guest_config.client_rx_port, IPPROTO_UDP, guest_config.mgmnt_iface); + rc = _ctrl.sock.agent_tx_sock->return_status; + if ( rc ) + { + elog ("Failed to setup 'guestAgent' transmitter\n" ); + return (rc) ; + } + + /******************************************************************/ + /* UDP Tx Message Socket Towards mtcAgent */ + /******************************************************************/ + + _ctrl.sock.mtc_event_tx_port = guest_config.hbs_to_mtc_event_port ; + _ctrl.sock.mtc_event_tx_sock = new msgClassTx(LOOPBACK_IP, guest_config.hbs_to_mtc_event_port, IPPROTO_UDP); + rc = _ctrl.sock.mtc_event_tx_sock->return_status; + if ( rc ) + { + elog ("Failed to setup 'mtcAgent' 'lo' transmitter on port (%d)\n", + _ctrl.sock.mtc_event_tx_port ); + + return (rc) ; + } + + /***************************************************************/ + /* Non-Blocking UDP Rx Message Socket for Maintenance Commands */ + /***************************************************************/ + + _ctrl.sock.mtc_cmd_port = guest_config.mtc_to_guest_cmd_port ; + _ctrl.sock.mtc_cmd_sock = new msgClassRx(LOOPBACK_IP, guest_config.mtc_to_guest_cmd_port, IPPROTO_UDP); + rc = _ctrl.sock.mtc_cmd_sock->return_status; + if ( rc ) + { + elog ("Failed to setup mtce command receive on port %d\n", + _ctrl.sock.mtc_cmd_port ); + return (rc) ; + } + + /* Get a socket that listens to the controller's FLOATING IP */ + /* This is the socket that the guestAgent receives events from + * the guestServer from the compute on */ + _ctrl.sock.agent_rx_float_sock = new msgClassRx(hostInv.hostBase.my_float_ip.c_str(), guest_config.agent_rx_port, IPPROTO_UDP); + rc = _ctrl.sock.agent_rx_float_sock->return_status; + if ( rc ) + { + elog ("Failed to setup 'guestServer' receiver on port %d\n", + _ctrl.sock.server_rx_port ); + return (rc) ; + } + + /* Get a socket that listens to the controller's LOCAL IP */ + /* This is the socket that the guestAgent receives events from + * the guestServer from the compute on */ + _ctrl.sock.agent_rx_local_sock = new msgClassRx(hostInv.hostBase.my_local_ip.c_str(), guest_config.agent_rx_port, IPPROTO_UDP); + rc = _ctrl.sock.agent_rx_local_sock->return_status; + if ( rc ) + { + elog ("Failed to setup 'guestServer' receiver on port %d\n", + _ctrl.sock.server_rx_port ); + return (rc) ; + } + + /* Don't fail the daemon if the logger port is not working */ + mtclogd_port_init (&_ctrl); + + rc = guestHttpSvr_init ( guest_config.vim_cmd_port ); + + return (rc) ; +} + +/* The main heartbeat service loop */ +int daemon_init ( string iface, string nodetype ) +{ + int rc = 10 ; + + /* Not used by this service */ + nodetype = nodetype ; + + /* Initialize socket construct and pointer to it */ + memset ( &_ctrl.sock, 0, sizeof(_ctrl.sock)); + + /* initialize the timer */ + mtcTimer_init ( _ctrl.timer ); + _ctrl.timer.hostname = "guestAgent" ; + + /* Assign interface to config */ + guest_config.mgmnt_iface = (char*)iface.data() ; + + httpUtil_init (); + + if ( daemon_files_init ( ) != PASS ) + { + elog ("Pid, log or other files could not be opened\n"); + rc = FAIL_FILES_INIT ; + } + + /* Bind signal handlers */ + else if ( daemon_signal_init () != PASS ) + { + elog ("daemon_signal_init failed\n"); + rc = FAIL_SIGNAL_INIT ; + } + + /* Configure the agent */ + else if ( (rc = daemon_configure ( )) != PASS ) + { + elog ("Daemon service configuration failed (rc:%i)\n", rc ); + rc = FAIL_DAEMON_CONFIG ; + } + + /* Setup the heartbeat service messaging sockets */ + else if ( (rc = _socket_init ( )) != PASS ) + { + elog ("socket initialization failed (rc:%d)\n", rc ); + rc = FAIL_SOCKET_INIT ; + } + else + { + _ctrl.timer.hostname = hostInv.hostBase.my_hostname ; + } + + return (rc); +} + + +/*************************************************************************** + * + * Name: send_cmd_to_guestServer + * + * Description: Messaging interface capable of building command specific + * messages and sending them to the guestServer daemon on + * the specified compute host. + * + * TODO: setup acknowledge mechanism using guestHost + * + ***************************************************************************/ +int send_cmd_to_guestServer ( string hostname, + unsigned int cmd, + string uuid, + bool reporting, + string event) +{ + mtc_message_type msg ; + int bytes_sent = 0 ; + int bytes_to_send = 0 ; + int rc = PASS ; + + string ip = hostInv.get_host_ip(hostname) ; + + + memset (&msg,0,sizeof(mtc_message_type)); + memcpy (&msg.hdr[0], get_guest_msg_hdr(), MSG_HEADER_SIZE ); + + /* Start creating the json string to he client */ + string payload = "{" ; + payload.append("\"source\":\""); + payload.append(hostInv.hostBase.my_float_ip); + payload.append("\""); + + if ( cmd == MTC_EVENT_LOOPBACK ) + { + ; /* go with default payload only */ + } + else if ( cmd == MTC_CMD_ADD_INST ) + { + ilog ("%s %s 'add' instance ; sent to guestServer\n", hostname.c_str(), uuid.c_str()); + payload.append(",\"uuid\":\""); + payload.append(uuid); + payload.append("\",\"service\":\"heartbeat\""); + payload.append(",\"state\":\""); + if ( reporting == true ) + payload.append("enabled\""); + else + payload.append("disabled\""); + } + else if ( cmd == MTC_CMD_DEL_INST ) + { + ilog ("%s %s 'delete' instance ; sent to guestServer\n", hostname.c_str(), uuid.c_str()); + payload.append(",\"uuid\":\""); + payload.append(uuid); + payload.append("\",\"service\":\"heartbeat\""); + payload.append(",\"state\":\"disabled\""); + } + else if ( cmd == MTC_CMD_MOD_INST ) + { + /* this may be a frequent log so its changed to a message log */ + mlog ("%s %s 'modify' instance ; sent to guestServer\n", hostname.c_str(), uuid.c_str()); + payload.append(",\"uuid\":\""); + payload.append(uuid); + payload.append("\",\"service\":\"heartbeat\""); + + payload.append(",\"state\":\""); + if ( reporting == true ) + payload.append("enabled\""); + else + payload.append("disabled\""); + } + else if ( cmd == MTC_CMD_MOD_HOST ) + { + payload.append(",\"uuid\":\""); + payload.append(uuid); + + /* In this host case , the instance heartbeat member + * contains the state we want to entire host to be + * put to */ + payload.append("\",\"heartbeat\":\""); + if ( reporting == true ) + payload.append("enabled\""); + else + payload.append("disabled\""); + + ilog ("%s %s 'modify' host (reporting=%s); sent to guestServer\n", + hostname.c_str(), + uuid.c_str(), + reporting ? "enabled" : "disabled" ); + } + else if ( cmd == MTC_CMD_QRY_INST ) + { + /* setting the query flag so the FSM + * knows to wait for the response. */ + hostInv.set_query_flag ( hostname ); + } + else if ( cmd == MTC_CMD_VOTE_INST + || cmd == MTC_CMD_NOTIFY_INST ) + { + bool vote ; + payload.append(",\"uuid\":\""); + payload.append(uuid); + payload.append("\",\"event\":\""); + payload.append(event); + payload.append("\""); + + if ( cmd == MTC_CMD_VOTE_INST ) + vote = true ; + else + vote = false ; + + ilog ("%s %s '%s' host (event=%s); sent to guestServer\n", + hostname.c_str(), + uuid.c_str(), + vote ? "vote" : "notify", + event.c_str()); + } + else + { + slog ("unsupported command (%d)\n", cmd ); + return (FAIL_BAD_CASE); + } + + payload.append("}"); + memcpy (&msg.buf[0], payload.data(), payload.length()); + msg.cmd = cmd ; + bytes_to_send = ((sizeof(mtc_message_type))-(BUF_SIZE))+(strlen(msg.buf)) ; + print_mtc_message ( &msg ); + bytes_sent = _ctrl.sock.agent_tx_sock->write((char *)&msg, bytes_to_send,ip.data(), _ctrl.sock.server_rx_port); + + if ( 0 > bytes_sent ) + { + elog("%s failed to send command (rc:%i)\n", hostname.c_str(), rc); + rc = FAIL_SOCKET_SENDTO ; + } + else if ( bytes_to_send != bytes_sent ) + { + wlog ("%s transmit byte count error (%d:%d)\n", + hostname.c_str(), bytes_to_send, bytes_sent ); + rc = FAIL_TO_TRANSMIT ; + } + else + { + mlog1 ("Transmit to %s port %5d\n", + _ctrl.sock.agent_tx_sock->get_dst_str(), + _ctrl.sock.agent_tx_sock->get_dst_addr()->getPort()); + rc = PASS ; + + /* Schedule receive ACK mechanism - bind a callback */ + } + + return (rc); +} + + +/*************************************************************************** + * + * Name: send_event_to_mtcAgent + * + * Description: Messaging interface capable of building the specified event + * messages and sending them to the mtcAgent daemon locally. + * + * TODO: setup acknowledge mechanism using guestHost + * + ***************************************************************************/ +int send_event_to_mtcAgent ( string hostname, + unsigned int event) +{ + int bytes_sent = 0 ; + int bytes_to_send = 0 ; + int rc = FAIL ; + + mtc_message_type msg ; + memset (&msg, 0 , sizeof(mtc_message_type)); + memcpy (&msg.hdr[0], get_guest_msg_hdr(), MSG_HEADER_SIZE ); + memcpy (&msg.hdr[MSG_HEADER_SIZE], "guestAgent", strlen("guestAgent")); + if ( event == MTC_EVENT_MONITOR_READY ) + { + if ( event == MTC_EVENT_MONITOR_READY ) + { + ilog ("%s requesting inventory from mtcAgent\n", hostname.c_str()); + } + msg.cmd = event ; + print_mtc_message (&msg ); + bytes_to_send = ((sizeof(mtc_message_type))-(BUF_SIZE))+(strlen(msg.buf)) ; + bytes_sent = _ctrl.sock.mtc_event_tx_sock->write((char*)&msg, bytes_to_send); + if ( 0 > bytes_sent ) + { + elog("%s Failed to send event (%d:%m)\n", hostname.c_str(), errno ); + rc = FAIL_SOCKET_SENDTO ; + } + else if ( bytes_to_send != bytes_sent ) + { + wlog ("%s transmit byte count error (%d:%d)\n", + hostname.c_str(), bytes_to_send, bytes_sent ); + rc = FAIL_TO_TRANSMIT ; + } + else + { + mlog1 ("Transmit to %s port %d\n", + _ctrl.sock.mtc_event_tx_sock->get_dst_str(), + _ctrl.sock.mtc_event_tx_sock->get_dst_addr()->getPort()); + rc = PASS ; + + /* Schedule receive ACK mechanism - bind a callback */ + } + } + else + { + slog ("Unsupported event (%d)\n", event ); + return ( FAIL_BAD_CASE ); + } + + return rc ; +} + +/*************************************************************************** + * + * Name: service_mtcAgent_command + * + * Description: Message handling interface capable of servicing mtcAgent + * commands such as 'add host', 'del host', 'mod host', etc. + * + * TODO: setup acknowledge mechanism using guestHost + * + ***************************************************************************/ +int service_mtcAgent_command ( unsigned int cmd , char * buf_ptr ) +{ + if ( !buf_ptr ) + { + slog ("Empty payload"); + return (FAIL); + } + string uuid = ""; + string hostname = ""; + string hosttype = ""; + string ip = ""; + + int rc = jsonUtil_get_key_val ( buf_ptr, "hostname", hostname ) ; + if ( rc != PASS ) + { + elog ("failed to get hostname\n"); + return (FAIL_GET_HOSTNAME); + } + rc = jsonUtil_get_key_val ( buf_ptr, "uuid", uuid ) ; + if ( rc != PASS ) + { + elog ("%s failed to get host 'uuid'\n", hostname.c_str()); + elog ("... buffer:%s\n", buf_ptr ); + + return (FAIL_INVALID_UUID); + } + + if ( cmd == MTC_CMD_ADD_HOST ) + { + rc = jsonUtil_get_key_val ( buf_ptr, "ip", ip ); + if ( rc == PASS ) + { + rc = jsonUtil_get_key_val ( buf_ptr, "personality", hosttype ); + if ( rc == PASS ) + { + rc = hostInv.add_host ( uuid, ip, hostname, hosttype ); + } + else + { + elog ("%s failed to get host 'personality'\n", hostname.c_str()); + } + } + else + { + elog ("%s failed to get host 'ip'\n", hostname.c_str()); + } + } + else if ( cmd == MTC_CMD_MOD_HOST ) + { + rc = jsonUtil_get_key_val ( buf_ptr, "ip", ip ); + if ( rc == PASS ) + { + rc = jsonUtil_get_key_val ( buf_ptr, "personality", hosttype ); + if ( rc == PASS ) + { + rc = hostInv.mod_host ( uuid, ip, hostname, hosttype ); + } + else + { + elog ("%s failed to get host 'personality'\n", hostname.c_str()); + } + } + else + { + elog ("%s failed to get host 'ip'\n", hostname.c_str()); + } + } + else if ( cmd == MTC_CMD_DEL_HOST ) + { + rc = hostInv.del_host ( uuid ); + } + else + { + wlog ("Unsupported command (%d)\n", cmd ); + rc = FAIL_BAD_CASE ; + } + return (rc); +} + + + +int recv_from_guestServer ( unsigned int cmd, char * buf_ptr ) +{ + int rc = PASS ; + switch ( cmd ) + { + case MTC_EVENT_MONITOR_READY: + { + string hostname = "" ; + if ( jsonUtil_get_key_val ( buf_ptr, "hostname", hostname ) ) + { + elog ("failed to extract 'hostname' from 'ready event'\n" ); + rc = FAIL_LOCATE_KEY_VALUE ; + } + else + { + ilog ("%s guestServer ready event\n", hostname.c_str()); + + /* Set all the instance state for this host */ + get_hostInv_ptr()->set_inst_state ( hostname ); + } + break ; + } + case MTC_EVENT_HEARTBEAT_RUNNING: + case MTC_EVENT_HEARTBEAT_STOPPED: + case MTC_EVENT_HEARTBEAT_ILLHEALTH: + { + string hostname = "" ; + string uuid = "" ; + int rc1 = jsonUtil_get_key_val ( buf_ptr, "hostname", hostname ); + int rc2 = jsonUtil_get_key_val ( buf_ptr, "uuid", uuid ); + if ( rc1 | rc2 ) + { + elog ("failed to parse 'hostname' or 'uuid' from heartbeat event buffer (%d:%d)\n", + rc1, rc2 ); + elog ("... Buffer: %s\n", buf_ptr ); + return ( FAIL_KEY_VALUE_PARSE ); + } + instInfo * instInfo_ptr = get_hostInv_ptr()->get_inst ( uuid ); + rc1 = guestUtil_get_inst_info ( hostname , instInfo_ptr, buf_ptr ); + if ( rc1 == PASS ) + { + if ( instInfo_ptr ) + { + string state ; + + if ( instInfo_ptr->heartbeat.reporting == true ) + state = "enabled" ; + else + state = "disabled" ; + + if ( cmd == MTC_EVENT_HEARTBEAT_ILLHEALTH ) + { + ilog ("%s %s ill health notification\n", hostname.c_str(), instInfo_ptr->uuid.c_str()); + rc = guestVimApi_alarm_event ( hostname, uuid ); + } + else if ( cmd == MTC_EVENT_HEARTBEAT_RUNNING ) + { + if ( instInfo_ptr->heartbeating != true ) + { + instInfo_ptr->heartbeating = true ; + ilog ("%s %s is now heartbeating\n", hostname.c_str(), instInfo_ptr->uuid.c_str()); + } + string status = "enabled"; + rc = guestVimApi_svc_event ( hostname, uuid, state, status, instInfo_ptr->restart_to_str); + } + else + { + if ( instInfo_ptr->heartbeating != false ) + { + instInfo_ptr->heartbeating = false ; + wlog ("%s %s is not heartbeating\n", hostname.c_str(), instInfo_ptr->uuid.c_str()); + } + string status = "disabled"; + rc = guestVimApi_svc_event ( hostname, uuid, state, status, "0"); + } + if ( rc != PASS ) + { + elog ("%s %s failed to send state change 'event' to vim (rc:%d)\n", + hostname.c_str(), instInfo_ptr->uuid.c_str(), rc ); + } + } + else + { + elog ("%s %s failed instance lookup\n", hostname.c_str(), uuid.c_str()); + } + } + else + { + elog ("failed to get instance info\n"); + } + break ; + } + + case MTC_EVENT_HEARTBEAT_LOSS: + { + string hostname = "" ; + string uuid = "" ; + int rc1 = jsonUtil_get_key_val ( buf_ptr, "hostname", hostname ) ; + int rc2 = jsonUtil_get_key_val ( buf_ptr, "uuid", uuid ) ; + if ( rc1 | rc2 ) + { + elog ("failed to parse 'heartbeat loss' key values (%d:%d)\n", rc1, rc2 ); + rc = FAIL_LOCATE_KEY_VALUE ; + } + else + { + if ( get_hostInv_ptr()->get_reporting_state ( hostname ) == true ) + { + instInfo * instInfo_ptr = get_hostInv_ptr()->get_inst ( uuid ) ; + if ( instInfo_ptr ) + { + if ( instInfo_ptr->heartbeat.reporting == true ) + { + rc = guestVimApi_inst_failed ( hostname, uuid , MTC_EVENT_HEARTBEAT_LOSS, 0 ); + } + else + { + ilog ("%s %s reporting disabled\n", hostname.c_str(), uuid.c_str() ); + } + } + else + { + elog ("%s %s failed instance lookup\n", hostname.c_str(), uuid.c_str() ); + rc = FAIL_HOSTNAME_LOOKUP ; + } + } + else + { + wlog ("%s heartbeat failure reporting disabled\n", hostname.c_str()); + } + } + break ; + } + case MTC_CMD_QRY_INST: + { + string hostname = "" ; + string uuid = "" ; + string status = "" ; + + jlog ("%s Instance Query Response: %s\n", hostname.c_str(), buf_ptr); + + int rc1 = jsonUtil_get_key_val ( buf_ptr, "hostname", hostname ) ; + int rc2 = jsonUtil_get_key_val ( buf_ptr, "uuid" , uuid ) ; + if ( rc1 | rc2 ) + { + ilog ("failed to parse 'hostname' or 'uuid' (%d:%d)\n", rc1, rc2 ); + } + + instInfo * instInfo_ptr = get_hostInv_ptr()->get_inst ( uuid ) ; + if ( instInfo_ptr ) + { + /** + * Verify that this instance is still associated with this host. + * This check was added as a fix and seeing + * a number of stale instance inventory in the guestServer. + * + * Without this check a late query response from an instance + * that was just deleted can result in an MOD sent to the + * server causing this instance to be mistakenly + * re-added to its inventory. + **/ + if ( !hostname.compare(instInfo_ptr->hostname) ) + { + /* + * Save the current reporting and heartbeating state + * only to compare to see if either has changed + */ + bool current_heartbeating_status = instInfo_ptr->heartbeating ; + bool current_reporting_state = instInfo_ptr->heartbeat.reporting ; + + if ( guestUtil_get_inst_info ( hostname, instInfo_ptr, buf_ptr ) == PASS ) + { + if ( instInfo_ptr->heartbeat.reporting != current_reporting_state ) + { + wlog ("%s:%s state mismatch\n", hostname.c_str(), uuid.c_str()); + wlog ("... state is '%s' but should be '%s' ... fixing\n", + instInfo_ptr->heartbeat.reporting ? "enabled" : "disabled", + current_reporting_state ? "enabled" : "disabled" ); + + instInfo_ptr->heartbeat.reporting = current_reporting_state ; + + rc = send_cmd_to_guestServer ( hostname, + MTC_CMD_MOD_INST , + uuid, + current_reporting_state ); + } + + if ( instInfo_ptr->heartbeating != current_heartbeating_status ) + { + string state ; + if ( instInfo_ptr->heartbeat.reporting == true ) + state = "enabled" ; + else + state = "disabled" ; + + if ( instInfo_ptr->heartbeating == true ) + { + string status = "enabled" ; + ilog ("%s %s is now heartbeating\n", hostname.c_str(), uuid.c_str()); + rc = guestVimApi_svc_event ( hostname, uuid, state, status, instInfo_ptr->restart_to_str); + } + else + { + string status = "disabled" ; + wlog ("%s %s is not heartbeating\n", hostname.c_str(), uuid.c_str()); + rc = guestVimApi_svc_event ( hostname, uuid, state, status, "0" ); + } + if ( rc != PASS ) + { + /* TODO: make this an elog before delivery */ + elog ("%s %s failed to send state change 'query' to vim (rc:%d)\n", hostname.c_str(), uuid.c_str(), rc ); + } + } + } + } + else + { + wlog ("%s %s no longer paired ; dropping query response\n", + hostname.c_str(), instInfo_ptr->uuid.c_str() ); + + /* Delete this just in case */ + send_cmd_to_guestServer ( hostname, MTC_CMD_DEL_INST , uuid, false ); + } + } + else + { + elog ("%s unknown uuid ; correcting ...\n", uuid.c_str() ); + + /* Delete this unknown host as it might somehow be stale */ + rc = send_cmd_to_guestServer ( hostname, MTC_CMD_DEL_INST , uuid, false ); + + rc = FAIL_UNKNOWN_HOSTNAME ; + } + hostInv.clr_query_flag ( hostname ); + break ; + } + case MTC_EVENT_VOTE_NOTIFY: + { + string hostname = "" ; + string instance_uuid = "" ; + string notification_type = ""; + string event = ""; + string vote = ""; + string reason = ""; + + int rc1 = jsonUtil_get_key_val ( buf_ptr, "hostname", hostname ) ; + int rc2 = jsonUtil_get_key_val ( buf_ptr, "uuid", instance_uuid ) ; + int rc3 = jsonUtil_get_key_val ( buf_ptr, "notification_type", notification_type ) ; + int rc4 = jsonUtil_get_key_val ( buf_ptr, "event-type", event ) ; + int rc5 = jsonUtil_get_key_val ( buf_ptr, "vote", vote ) ; + if ( rc1 | rc2 | rc3 | rc4 | rc5 ) + { + elog ("failed to parse 'vote-notify' key values (%d:%d:%d:%d:%d)\n", rc1, rc2, rc3, rc4, rc5); + rc = FAIL_LOCATE_KEY_VALUE ; + } + else + { + // 'reason' is optional + jsonUtil_get_key_val ( buf_ptr, "reason", reason ) ; + + jlog ("%s Instance Vote/Notification Response: %s\n", instance_uuid.c_str(), buf_ptr); + + string guest_response = ""; + + if (!vote.compare("accept") || !vote.compare("complete")) + { + if (!notification_type.compare("revocable")) + { + guest_response = "allow"; + } + else if (!notification_type.compare("irrevocable")) + { + guest_response = "proceed"; + } + else + { + rc = FAIL_BAD_PARM; + break; + } + } + else if (!vote.compare("reject")) + { + guest_response = "reject"; + } + else + { + rc = FAIL_BAD_PARM; + } + guestVimApi_inst_action (hostname, instance_uuid, event, guest_response, reason); + } + break ; + } + default: + elog ("Unsupported comand (%d)\n", cmd ); + } + return (rc); +} + + +void guestHostClass::run_fsm ( string hostname ) +{ + guestHostClass::guest_host * guest_host_ptr ; + guest_host_ptr = guestHostClass::getHost ( hostname ); + if ( guest_host_ptr != NULL ) + { + /* This FSM is only run on computes */ + if (( guest_host_ptr->hosttype & COMPUTE_TYPE ) == COMPUTE_TYPE) + { + flog ("%s FSM\n", hostname.c_str() ); + } + } +} + +/* Top level call to run FSM */ +/* TODO: Deal with delete */ +int guest_fsm_run ( guestHostClass * obj_ptr ) +{ + instInfo instance ; + + /* Run Maintenance on Inventory */ + for ( obj_ptr->hostlist_iter_ptr = obj_ptr->hostlist.begin () ; + obj_ptr->hostlist_iter_ptr != obj_ptr->hostlist.end () ; + obj_ptr->hostlist_iter_ptr++ ) + { + string hostname = *obj_ptr->hostlist_iter_ptr ; + + daemon_signal_hdlr (); + + obj_ptr->run_fsm ( hostname ); + + /* Run the audit on each host */ + if ( obj_ptr->audit_timer.ring == true ) + { + // ilog ("%s FSM Audit !\n", hostname.c_str() ); + + obj_ptr->audit_run = true ; + + if ( obj_ptr->get_got_host_state ( hostname ) == false ) + { + libEvent & event = hostInv.get_host_event ( hostname ); + string uuid = obj_ptr->get_host_uuid (hostname) ; + int rc = guestVimApi_getHostState ( hostname, uuid, event ); + + if ( rc != PASS ) + { + wlog ("%s failed to get host level reporting state (rc=%d)\n", + hostname.c_str(), rc); + } + else + { + /* Only set it if true as it is defaulted to false already. + * The VIM will send an enable command at a later time */ + if ( !event.value.compare("enabled")) + { + ilog ("%s fault reporting enabled\n", hostname.c_str()); + + rc = hostInv.set_reporting_state ( hostname, true ); + if ( rc != PASS ) + { + wlog ("%s failed to set host level reporting state (rc=%d)\n", + hostname.c_str(), rc); + } + } + else + { + rc = hostInv.set_reporting_state ( hostname, false ); + } + + dlog ("%s Got host state\n", hostname.c_str() ); + obj_ptr->set_got_host_state ( hostname ); + } + } + + /* make sure that the instances for this host are loaded */ + if ( obj_ptr->get_got_instances ( hostname ) == false ) + { + libEvent & event = hostInv.get_host_event ( hostname ); + string uuid = obj_ptr->get_host_uuid (hostname) ; + int rc = guestVimApi_getHostInst ( hostname, uuid, event ); + if ( rc != PASS ) + { + wlog ("%s failed to get host instances (rc=%d)\n", hostname.c_str(), rc); + } + else + { + obj_ptr->set_got_instances ( hostname ); + dlog ("%s instances loaded\n", hostname.c_str() ); + } + } + + /* only query the guestServer if reporting for that server + * is 'enabled' and instance list is not empty */ + if (( obj_ptr->num_instances ( hostname ) != 0 ) && + ( obj_ptr->get_reporting_state ( hostname ) == true )) + { + if ( obj_ptr->get_query_flag ( hostname ) == true ) + { + obj_ptr->inc_query_misses ( hostname); + dlog ("%s guestServer Query Misses:%d\n", hostname.c_str(), + obj_ptr->get_query_misses ( hostname )); + } + else + { + obj_ptr->clr_query_misses ( hostname ); + } + + /* Note: The 3rd and 4th parms are not needed + * for the MTC_CMD_QRY_INST command */ + send_cmd_to_guestServer ( hostname, MTC_CMD_QRY_INST, "", false ); + } + } + if ( obj_ptr->exit_fsm == true ) + { + obj_ptr->exit_fsm = false ; + break ; + } + } + if (( obj_ptr->audit_timer.ring == true ) && ( obj_ptr->audit_run == true )) + { + // dlog ("Audit Restarted\n"); + obj_ptr->audit_run = false ; + obj_ptr->audit_timer.ring = false ; + mtcTimer_start ( obj_ptr->audit_timer , _timer_handler, 10 ); + } + + return ( PASS ); +} + +/*****************************************************************************/ +/*****************************************************************************/ +/*****************************************************************************/ + +void daemon_service_run ( void ) +{ + int rc = PASS ; + int count = 0 ; + int flush_thld = 0 ; + + mtcTimer_start ( hostInv.audit_timer , _timer_handler, 2 ); + + guestHttpUtil_init (); + + /* socket descriptor list */ + std::list socks ; + + /* Not monitoring address changes RTMGRP_IPV4_IFADDR | RTMGRP_IPV6_IFADDR */ + if (( _ctrl.sock.ioctl_sock = open_ioctl_socket ( )) <= 0 ) + { + elog ("Failed to create ioctl socket"); + daemon_exit (); + } + + socks.clear(); + socks.push_front (_ctrl.sock.agent_rx_local_sock->getFD()); + socks.push_front (_ctrl.sock.agent_rx_float_sock->getFD()); + socks.push_front (_ctrl.sock.mtc_cmd_sock->getFD()); + dlog ("Selects: %d %d %d\n", _ctrl.sock.agent_rx_local_sock->getFD(), + _ctrl.sock.agent_rx_float_sock->getFD(), + _ctrl.sock.mtc_cmd_sock->getFD()); + socks.sort(); + + ilog ("Sending ready event to maintenance\n"); + do + { + /* Wait for maintenance */ + rc = send_event_to_mtcAgent ( hostInv.hostBase.my_hostname, + MTC_EVENT_MONITOR_READY ) ; + if ( rc == RETRY ) + { + mtcWait_secs ( 3 ); + } + } while ( rc == RETRY ) ; + + if ( rc == FAIL ) + { + elog ("Unrecoverable heartbeat startup error (rc=%d)\n", rc ); + daemon_exit (); + } + + /* enable the base level signal handler latency monitor */ + daemon_latency_monitor (true); + + ilog ("------------------------------------------------------------\n"); + + for ( ;; ) + { + /* Service Sockets */ + hostInv.waitd.tv_sec = 0; + hostInv.waitd.tv_usec = GUEST_SOCKET_TO ; + + /* Initialize the master fd_set */ + FD_ZERO(&hostInv.message_readfds); + + FD_SET(_ctrl.sock.agent_rx_local_sock->getFD(), &hostInv.message_readfds); + FD_SET(_ctrl.sock.agent_rx_float_sock->getFD(), &hostInv.message_readfds); + FD_SET(_ctrl.sock.mtc_cmd_sock->getFD(), &hostInv.message_readfds); + + /* Call select() and wait only up to SOCKET_WAIT */ + rc = select( socks.back()+1, &hostInv.message_readfds, NULL, NULL, &hostInv.waitd); + if (( rc < 0 ) || ( rc == 0 ) || ( rc > (int)socks.size())) + { + /* Check to see if the select call failed. */ + /* ... but filter Interrupt signal */ + if (( rc < 0 ) && ( errno != EINTR )) + { + wlog_throttled ( count, 20, "socket select failed (%d:%m)\n", errno); + } + else if ( rc > (int)socks.size()) + { + wlog_throttled ( count, 100, "Select return exceeds current file descriptors (%ld:%d)\n", + socks.size(), rc ); + } + else + { + count = 0 ; + } + } + else + { + mtc_message_type msg ; + memset ((void*)&msg,0,sizeof(mtc_message_type)); + + /* Service guestServer messages towards the local IP */ + if (FD_ISSET(_ctrl.sock.agent_rx_local_sock->getFD(), &hostInv.message_readfds) ) + { + int bytes = _ctrl.sock.agent_rx_local_sock->read((char*)&msg.hdr[0], sizeof(mtc_message_type)); + + mlog1 ("Received %d bytes from %s:%d:guestServer (local)\n", bytes, + _ctrl.sock.agent_rx_local_sock->get_src_str(), + _ctrl.sock.agent_rx_local_sock->get_dst_addr()->getPort()); + + recv_from_guestServer ( msg.cmd, msg.buf ); + print_mtc_message ( &msg ); + } + + /* Service guestServer messages towards the floating IP */ + else if (FD_ISSET(_ctrl.sock.agent_rx_float_sock->getFD(), &hostInv.message_readfds) ) + { + int bytes = _ctrl.sock.agent_rx_float_sock->read((char*)&msg.hdr[0], sizeof(mtc_message_type)); + + mlog1 ("Received %d bytes from %s:%d:guestServer (float)\n", bytes, + _ctrl.sock.agent_rx_float_sock->get_src_str(), + _ctrl.sock.agent_rx_port); + + recv_from_guestServer ( msg.cmd, msg.buf ); + print_mtc_message ( &msg ); + } + + /* Service mtcAgent commands */ + else if (FD_ISSET(_ctrl.sock.mtc_cmd_sock->getFD(), &hostInv.message_readfds) ) + { + int bytes = _ctrl.sock.mtc_cmd_sock->read((char*)&msg.hdr[0],sizeof(mtc_message_type)); + + mlog1 ("Received %d bytes from %s:%d:mtcAgent\n", bytes, + _ctrl.sock.mtc_cmd_sock->get_src_str(), + _ctrl.sock.mtc_cmd_port); + print_mtc_message ( &msg ); + if ( !strncmp ( get_cmd_req_msg_header(), &msg.hdr[0], MSG_HEADER_SIZE )) + { + service_mtcAgent_command ( msg.cmd , &msg.buf[0] ); + count = 0 ; + } + else + { + wlog_throttled ( count, 100, "Invalid message header\n"); + } + } + else + { + ilog ("Unknown select\n"); + } + } + + guestHttpSvr_look (); + + guest_fsm_run ( &hostInv ) ; + + daemon_signal_hdlr (); + + /* Support the log flush config option */ + if ( guest_config.flush ) + { + if ( ++flush_thld > guest_config.flush_thld ) + { + flush_thld = 0 ; + fflush (stdout); + fflush (stderr); + } + } + } + daemon_exit (); +} + +/* Push daemon state to log file */ +void daemon_dump_info ( void ) +{ + daemon_dump_membuf_banner (); + + hostInv.print_node_info (); + hostInv.memDumpAllState (); + + daemon_dump_membuf(); +} + +const char MY_DATA [100] = { "eieio\n" } ; +const char * daemon_stream_info ( void ) +{ + return (&MY_DATA[0]); +} + +/** Teat Head Entry */ +int daemon_run_testhead ( void ) +{ + int rc = PASS; + return (rc); +} diff --git a/mtce-guest/src/guestBase.h b/mtce-guest/src/guestBase.h new file mode 100644 index 00000000..2381ccdb --- /dev/null +++ b/mtce-guest/src/guestBase.h @@ -0,0 +1,319 @@ +#ifndef __INCLUDE_GUESTBASE_H__ +#define __INCLUDE_GUESTBASE_H__ + +/* + * Copyright (c) 2013-2016 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + + /** + * @file + * Wind River CGTS Platform Guest Services "Base" Header + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace std; + +#include "msgClass.h" +#include "nodeBase.h" +#include "httpUtil.h" +#include "nodeTimers.h" + +#define WANT_NEW + +/** + * @addtogroup guest_services_base + * @{ + */ + +#ifdef __AREA__ +#undef __AREA__ +#endif +#define __AREA__ "gst" + +#define CONFIG_CLIENT_RX_PORT (0x00000001) +#define CONFIG_MTC_EVENT_PORT (0x00000002) +#define CONFIG_MTC_CMD_PORT (0x00000004) +#define CONFIG_AGENT_RX_PORT (0x00000008) +#define CONFIG_VIM_CMD_RX_PORT (0x00000010) +#define CONFIG_VIM_EVENT_RX_PORT (0x00000020) + +#define HB_DEFAULT_FIRST_MS 2000 +#define HB_DEFAULT_INTERVAL_MS 1000 +#define HB_DEFAULT_REBOOT_MS 10000 +#define HB_DEFAULT_VOTE_MS 10000 +#define HB_DEFAULT_SHUTDOWN_MS 10000 +#define HB_DEFAULT_SUSPEND_MS 10000 +#define HB_DEFAULT_RESUME_MS 10000 +#define HB_DEFAULT_RESTART_MS 120000 + +/* Directory where libvirt creates the serial I/O pipe channel sockets into the guest + * We monitor this directory with inotify for file changes */ +#define QEMU_CHANNEL_DIR ((const char *)"/var/lib/libvirt/qemu") + +#define ARRAY_SIZE(x) ((int)(sizeof(x)/sizeof(*x))) + +#define MAX_INSTANCES (100) +#define MAX_MESSAGES (10) + +/* The socket select timeout */ +#define GUEST_SOCKET_TO (10000) + +#define DEFAULT_CONNECT_WAIT (1) + +#define CONNECT_TIMOUT (60) +#define WAIT_FOR_INIT_TIMEOUT (60) +#define HEARTBEAT_START_TIMEOUT (120) +#define SEARCH_AUDIT_TIME (180) + +void guestTimer_handler ( int sig, siginfo_t *si, void *uc); + +const char * get_guest_msg_hdr (void) ; + +typedef struct +{ + char buffer [256]; +} gst_message_type ; + +typedef enum +{ + hbs_invalid, + hbs_server_waiting_init, + hbs_server_waiting_challenge, + hbs_server_waiting_response, + hbs_server_paused, // heartbeat paused at request of vm + hbs_server_nova_paused, // heartbeat paused at request of nova + hbs_server_migrating, // heartbeat paused while migrate in progress + hbs_server_corrective_action, + hbs_client_waiting_init_ack, + hbs_client_waiting_challenge, + hbs_client_waiting_pause_ack, + hbs_client_waiting_resume_ack, + hbs_client_paused, + hbs_client_waiting_shutdown_ack, + hbs_client_waiting_shutdown_response, + hbs_client_shutdown_response_recieved, + hbs_client_exiting, + hbs_state_max +} hb_state_t; + +/** Guest service control messaging socket control structure */ +typedef struct +{ + /** Guest Services Messaging Agent Receive (from guestServer) Socket + * + * Note: This socket supports receiving from the computes specifying + * either the floating or local IP */ + int agent_rx_port ; + msgClassSock* agent_rx_float_sock ; + msgClassSock* agent_rx_local_sock ; + + /** Guest Services Messaging Agent Transmit (to guestServer) Socket + * + * Note: This transmit socket can be used for any port + * specified at send time */ + msgClassSock* agent_tx_sock ; + + + /** Guest Services Messaging Socket mtcAgent commands are received on */ + msgClassSock* mtc_cmd_sock ; + int mtc_cmd_port ; + + /** Guest Services Messaging Server Receive (from guestAgent) Socket */ + msgClassSock* server_rx_sock ; + int server_rx_port ; + + /** Guest Services Messaging Server Transmit (to guestAgent) Socket */ + msgClassSock* server_tx_sock ; + struct sockaddr_in server_tx_addr ; + + /** Socket used to transmit READY status and Events to Maintenance */ + int mtc_event_tx_port ; + msgClassSock* mtc_event_tx_sock ; + + int netlink_sock ; /* netlink socket */ + int ioctl_sock ; /* general ioctl socket */ + + msgSock_type mtclogd ; +} guest_services_socket_type ; + +/** + * The HTTP server supports two URL levels ; + * a hosts level and instances level. + **/ +typedef enum +{ + SERVICE_LEVEL_NONE, + SERVICE_LEVEL_HOST, + SERVICE_LEVEL_INST, +} service_level_enum ; + +/** common service_type control info */ +typedef struct +{ + bool provisioned ; /* set true once the VIM issues create */ + string state ; /* enabled, configured or disabled */ + bool reporting ; /* failue reporting state */ + + int failures ; /* Running count of failures */ + bool failed ; /* true means heartbeating has failed */ + bool waiting ; /* Waiting on a response */ + int b2b_misses ; /* running back-to-back misses */ +} service_type ; + +/** A grouping of info extracted from command's url */ +typedef struct +{ + service_level_enum service_level ; + string uuid ; + string command ; + string temp ; +} url_info_type ; + +/** instance control structure */ +typedef struct +{ + string hostname ; /**< The host that this instance is on */ + + /* Instance identifiers */ + string name ; /**< the Instance Name as it appears in the GUI */ + string uuid ; /**< the instance uuid which is unique to the system */ + string chan ; /**< virtio channel name 'cgcs.heartbeat..sock' */ + string inst ; /**< the instance part of the channel name */ + + /* Set to true when this channel has been provisioned by the guestAgent */ + // bool provisioned ; + + /* + * Full path and name to the detected channel. + * Used to set inotify file watch. + */ + string fd_namespace ; + + #define CHAN_FLAGS (SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC ) + int chan_fd ; + bool chan_ok ; + + bool connecting ; + bool connected ; /* true = the channel is connected to the guest */ + bool heartbeating ; /* true = the heartbeating has started */ + + string name_log_prefix ; + string uuid_log_prefix ; + + int connect_wait_in_secs ; + + /* added service bools */ + service_type heartbeat ; + service_type reserved ; + + /* + * File and watch descriptors used to monitor + * specific files in QEMU_CHANNEL_DIR + */ + int inotify_file_fd ; + int inotify_file_wd ; + + /* Message header info */ + int version; + int revision; + string msg_type; + uint32_t sequence; + + hb_state_t hbState ; /* see heartbeat_types.h */ + hb_state_t vnState ; /* see heartbeat_types.h */ + + uint32_t invocation_id ; + + // For voting and notification + string event_type; // GuestHeartbeatMsgEventT + string notification_type; // GuestHeartbeatMsgNotifyT + + uint32_t heartbeat_challenge ; + uint32_t heartbeat_interval_ms ; + + uint32_t vote_secs; + uint32_t shutdown_notice_secs; + uint32_t suspend_notice_secs; + uint32_t resume_notice_secs; + uint32_t restart_secs; + string corrective_action; + + string unhealthy_corrective_action; + bool unhealthy_failure ; + + /* String versions of the above timeouts - integer portions only */ + string vote_to_str ; /* vote timeout in seconds as a string value */ + string shutdown_to_str ; /* shutdown timeout in seconds as a string value */ + string suspend_to_str ; /* suspend timeout in seconds as a string value */ + string resume_to_str ; /* resume timeout in seconds as a string value */ + string restart_to_str ; /* restart timeout in seconds as a string value */ + + int select_count ; + int message_count ; + int health_count ; + int failure_count ; + int connect_count ; + int connect_retry_count ; + int corrective_action_count ; + + libEvent vimEvent ; + +} instInfo ; + +/* daemon control structure - used for both guestAgent and guestServer */ +typedef struct +{ + bool init ; + char hostname [MAX_HOST_NAME_SIZE+1]; + string address ; + string address_peer ; /* used for server only */ + int nodetype ; /* used for server only */ + + guest_services_socket_type sock ; + struct mtc_timer timer ; + + /* List of instances provisioned on this host */ + list instance_list ; /* used for server only */ + list::iterator instance_list_ptr; /* used for server only */ + + /* file and watch descriptors used to monitor QEMU_CHANNEL_DIR */ + int inotify_dir_fd ; + int inotify_dir_wd ; + + + +} ctrl_type ; + +ctrl_type * get_ctrl_ptr ( void ); + + + +int send_cmd_to_guestServer ( string hostname, unsigned int cmd, string uuid, bool reporting, string event="unknown" ); + +/** + * @} guest_services_base + */ + +#endif /* __INCLUDE_GUESTBASE_H__ */ diff --git a/mtce-guest/src/guestClass.cpp b/mtce-guest/src/guestClass.cpp new file mode 100644 index 00000000..69cf64f7 --- /dev/null +++ b/mtce-guest/src/guestClass.cpp @@ -0,0 +1,1335 @@ +/* + * Copyright (c) 2013-2016 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + + /** + * @file + * Wind River CGTS Platform Guest Services "Base" Header + */ + +#include "nodeBase.h" +#include "nodeTimers.h" +#include "guestClass.h" +#include "nodeUtil.h" + + +const char guest_msg_hdr [MSG_HEADER_SIZE] = {"guest msg header:"}; +const char * get_guest_msg_hdr (void) { return guest_msg_hdr ; } + +/* used as a default return in procedures that return a reference to libEvent */ +libEvent nullEvent ; + +/**< constructor */ +guestHostClass::guestHostClass() +{ + guest_head = guest_tail = NULL; + memory_allocs = 0 ; + memory_used = 0 ; + hosts = 0 ; + + for ( int i = 0 ; i < MAX_HOSTS ; i++ ) + { + host_ptrs[i] = NULL ; + } + /* Query Host state from the VIM bools */ + audit_run = false ; + + exit_fsm = false ; + +// httpUtil_event_init ( &nullEvent, "null", "null" , "0.0.0.0", 0 ); + nullEvent.request = SERVICE_NONE ; + return ; +} + +/**< destructor */ +guestHostClass::~guestHostClass() +{ + guest_host * ptr = guest_head ; + guest_host * temp_ptr = ptr ; + while ( ptr != NULL ) + { + temp_ptr = ptr ; + ptr = ptr->next ; + delHost (temp_ptr); + } + if ( memory_used != 0 ) + { + elog ( "Apparent Memory Leak - Allocs:%d and Bytes:%d\n", + memory_allocs, memory_used ); + } + else + { + dlog ( "No Memory Leaks\n\n"); + } + return ; +} + +/* + * Allocate new host and tack it on the end of the host_list + */ +struct guestHostClass::guest_host* guestHostClass::addHost( string hostname ) +{ + /* verify host is not already provisioned */ + struct guest_host * ptr = guestHostClass::getHost ( hostname ); + if ( ptr ) + { + if ( guestHostClass::remHost ( hostname ) ) + { + /* Should never get here but if we do then */ + /* something is seriously wrong */ + elog ("Error: Unable to remove host during reprovision\n"); + return static_cast(NULL); + } + } + + /* allocate memory for new host */ + ptr = guestHostClass::newHost (); + if( ptr == NULL ) + { + elog ( "Error: Failed to allocate memory for new host\n" ); + return static_cast(NULL); + } + + /* Init the new host */ + ptr->hostname = hostname ; + ptr->reporting = false ; + ptr->query_flag = false ; + + ptr->got_host_state = false; + ptr->got_instances = false; + + ptr->stage = STAGE__START ; + ptr->instance_list.clear(); + + /* Init host's general mtc timer */ + mtcTimer_init ( ptr->host_audit_timer ); + + /* Assign the timer the host's name */ + ptr->host_audit_timer.hostname = hostname ; + + /* If the host list is empty add it to the head */ + if( guest_head == NULL ) + { + guest_head = ptr ; + guest_tail = ptr ; + ptr->prev = NULL ; + ptr->next = NULL ; + } + else + { + /* link the new_host to the tail of the host_list + * then mark the next field as the end of the host_list + * adjust tail to point to the last host + */ + guest_tail->next = ptr ; + ptr->prev = guest_tail ; + ptr->next = NULL ; + guest_tail = ptr ; + } + + hosts++ ; + dlog2 ("Added guestHostClass host instance %d\n", hosts); + return ptr ; +} + +/* Remove a hist from the linked list of hosts - may require splice action */ +int guestHostClass::remHost( string hostname ) +{ + if ( hostname.c_str() == NULL ) + return -ENODEV ; + + if ( guest_head == NULL ) + return -ENXIO ; + + struct guest_host * ptr = guestHostClass::getHost ( hostname ); + + if ( ptr == NULL ) + return -EFAULT ; + + /* Free the mtc timer if in use */ + if ( ptr->host_audit_timer.tid ) + { + tlog ("%s Stopping host timer\n", hostname.c_str()); + mtcTimer_stop ( ptr->host_audit_timer ); + ptr->host_audit_timer.ring = false ; + ptr->host_audit_timer.tid = NULL ; + } + + /* If the host is the head host */ + if ( ptr == guest_head ) + { + /* only one host in the list case */ + if ( guest_head == guest_tail ) + { + dlog ("Single Host -> Head Case\n"); + guest_head = NULL ; + guest_tail = NULL ; + } + else + { + dlog ("Multiple Hosts -> Head Case\n"); + guest_head = guest_head->next ; + guest_head->prev = NULL ; + } + } + /* if not head but tail then there must be more than one + * host in the list so go ahead and chop the tail. + */ + else if ( ptr == guest_tail ) + { + dlog ("Multiple Host -> Tail Case\n"); + guest_tail = guest_tail->prev ; + guest_tail->next = NULL ; + } + else + { + dlog ("Multiple Host -> Full Splice Out\n"); + ptr->prev->next = ptr->next ; + ptr->next->prev = ptr->prev ; + } + guestHostClass::delHost ( ptr ); + hosts-- ; + return (PASS) ; +} + + +struct guestHostClass::guest_host* guestHostClass::getHost ( string hostname_or_uuid ) +{ + for ( struct guest_host * ptr = guest_head ; guest_head ; ptr = ptr->next ) + { + if ( !hostname_or_uuid.compare ( ptr->hostname )) + { + return ptr ; + } + else if ( !hostname_or_uuid.compare ( ptr->uuid )) + { + return ptr ; + } + + if (( ptr->next == NULL ) || ( ptr == guest_tail )) + break ; + } + return static_cast(NULL); +} + +/* + * Allocates memory for a new host and stores its the address in host_ptrs + * + * @param void + * @return pointer to the newly allocted host memory + */ +struct guestHostClass::guest_host * guestHostClass::newHost ( void ) +{ + struct guestHostClass::guest_host * temp_host_ptr = NULL ; + + if ( memory_allocs == 0 ) + { + memset ( host_ptrs, 0 , sizeof(struct guest_host *)*MAX_HOSTS); + } + + // find an empty spot + for ( int i = 0 ; i < MAX_HOSTS ; i++ ) + { + if ( host_ptrs[i] == NULL ) + { + host_ptrs[i] = temp_host_ptr = new guest_host ; + memory_allocs++ ; + memory_used += sizeof (struct guestHostClass::guest_host); + + return temp_host_ptr ; + } + } + elog ( "Failed to save new host pointer address\n" ); + return temp_host_ptr ; +} + +/* Frees the memory of a pre-allocated host and removes + * it from the host_ptrs list + * @param host * pointer to the host memory address to be freed + * @return int return code { PASS or -EINVAL } + */ +int guestHostClass::delHost ( struct guestHostClass::guest_host * host_ptr ) +{ + if ( guestHostClass::memory_allocs > 0 ) + { + for ( int i = 0 ; i < MAX_NODES ; i++ ) + { + if ( guestHostClass::host_ptrs[i] == host_ptr ) + { + delete host_ptr ; + guestHostClass::host_ptrs[i] = NULL ; + guestHostClass::memory_allocs-- ; + guestHostClass::memory_used -= sizeof (struct guestHostClass::guest_host); + return PASS ; + } + } + elog ( "Error: Unable to validate memory address being freed\n" ); + } + else + elog ( "Error: Free memory called when there is no memory to free\n" ); + + return -EINVAL ; +} + + +int guestHostClass::mod_host ( string uuid, string address, string hostname, string hosttype ) +{ + struct guestHostClass::guest_host * host_ptr = static_cast(NULL); + + if (hostname.empty()) + { + wlog ("Refusing to modify host with 'null' or 'invalid' hostname (uuid:%s)\n", + uuid.c_str()); + return (FAIL_INVALID_HOSTNAME) ; + } + + host_ptr = guestHostClass::getHost(hostname); + if ( !host_ptr ) + { + ilog ("%s not already provisioned\n", host_ptr->hostname.c_str()); + + /* Send back a retry in case the add needs to be converted to a modify */ + return (FAIL_INVALID_OPERATION); + } + host_ptr->uuid = uuid ; + host_ptr->ip = address ; + host_ptr->hosttype = get_host_function_mask (hosttype) ; + + ilog ("%s modify %s %s %s\n", + hostname.c_str(), + host_ptr->uuid.c_str(), + host_ptr->ip.c_str(), + hosttype.c_str()); + + return (PASS); +} + +int guestHostClass::add_host ( string uuid, + string address, + string hostname, + string hosttype) +{ + int rc = FAIL ; + struct guestHostClass::guest_host * host_ptr = static_cast(NULL); + + host_ptr = guestHostClass::getHost(hostname); + if ( host_ptr ) + { + ilog ("%s Already provisioned\n", host_ptr->hostname.c_str()); + + /* Send back a retry in case the add needs to be converted to a modify */ + return (RETRY); + } + /* Otherwise add it as a new host */ + else + { + host_ptr = guestHostClass::addHost(hostname); + if ( host_ptr ) + { + host_ptr->uuid = uuid ; + host_ptr->ip = address ; + host_ptr->hosttype = get_host_function_mask(hosttype); + + mtcTimer_init ( host_ptr->host_audit_timer, hostname ); + + host_ptr->stage = STAGE__START ; + + /* Add to the end of inventory */ + hostlist.push_back ( host_ptr->hostname ); + + rc = PASS ; + ilog ("%s added\n", hostname.c_str()); + } + else + { + elog ("%s add failed\n", hostname.c_str()); + rc = FAIL_NULL_POINTER ; + } + } + return (rc); +} + +/***************************************************************************** + * + * Name : rem_host + * + * Purpose : Remove this host from daemon all together + * + *****************************************************************************/ +int guestHostClass::rem_host ( string hostname ) +{ + int rc = FAIL ; + if ( ! hostname.empty() ) + { + /* remove the service specific component */ + hostlist.remove ( hostname ); + + exit_fsm = true ; + + /* free memory */ + rc = guestHostClass::remHost ( hostname ); + } + return ( rc ); +} + +/***************************************************************************** + * + * Name : del_host_inst + * + * Purpose : Delete all instances for this host + * + *****************************************************************************/ +int guestHostClass::del_host_inst ( string host_uuid ) +{ + int rc = FAIL_DEL_UNKNOWN ; + guestHostClass::guest_host * guest_host_ptr = guestHostClass::getHost( host_uuid ); + if ( guest_host_ptr ) + { + if ( guest_host_ptr->instance_list.size() != 0 ) + { + for ( guest_host_ptr->instance_list_ptr = guest_host_ptr->instance_list.begin(); + guest_host_ptr->instance_list_ptr != guest_host_ptr->instance_list.end(); + guest_host_ptr->instance_list_ptr++ ) + { + send_cmd_to_guestServer ( guest_host_ptr->hostname, + MTC_CMD_DEL_INST, + guest_host_ptr->instance_list_ptr->uuid, + guest_host_ptr->instance_list_ptr->heartbeat.reporting ); + } + + /* If the instance list is empty then clear the query flag */ + if ( guest_host_ptr->instance_list.empty () ) + { + clr_query_flag ( guest_host_ptr->hostname ); + } + } + } + else + { + wlog ("Unknown host uuid: %s\n", host_uuid.c_str()); + } + return (rc); +} + +/***************************************************************************** + * + * Name : del_inst + * + * Purpose : Add an instance to Delete all instances and then the host + * + *****************************************************************************/ +int guestHostClass::del_host ( string uuid ) +{ + int rc = FAIL_DEL_UNKNOWN ; + guestHostClass::guest_host * guest_host_ptr = guestHostClass::getHost( uuid ); + if ( guest_host_ptr ) + { + if ( guest_host_ptr->instance_list.size() != 0 ) + { + for ( guest_host_ptr->instance_list_ptr = guest_host_ptr->instance_list.begin(); + guest_host_ptr->instance_list_ptr != guest_host_ptr->instance_list.end(); + guest_host_ptr->instance_list_ptr++ ) + { + send_cmd_to_guestServer ( guest_host_ptr->hostname, + MTC_CMD_DEL_INST, + guest_host_ptr->instance_list_ptr->uuid, + guest_host_ptr->instance_list_ptr->heartbeat.reporting ); + } + } + /* save the hostname so that the logs below refer to something valid */ + string hostname = guest_host_ptr->hostname ; + rc = rem_host ( hostname ); + if ( rc == PASS ) + { + ilog ("%s deleted\n", hostname.c_str()); + print_node_info(); + } + else + { + elog ("%s delete host failed (rc:%d)\n", hostname.c_str(), rc ); + } + } + else + { + wlog ("Unknown uuid: %s\n", uuid.c_str()); + } + return (rc); +} + +/***************************************************************************** + * + * Name : add_inst + * + * Purpose : Add an instance to the guestAgent guestHostClass database + * + * Assumptions: This acts as a modify as well. See description below. + * + * Description: Search through all guestAgent guestHostClass database Looking + * for the specified instance uuid. + * + * If found against the specified hostname then just ensure the + * channel info is correct or updated, the services are disabled + * and the counters are reset. + * + * If found against a different host the do the same + * initialization but add a unique log staing this condition. + * + * If not found then just go ahead and add it to the instance + * list for the specified host with the same default initialization. + * + * Send a add command to the guestServer so that it has the + * opportunity to try to open the channel to the guest and + * start heartbeating. + * + *****************************************************************************/ + +int guestHostClass::add_inst ( string hostname, instInfo & instance ) +{ + int rc = FAIL_NOT_FOUND ; + + if ( instance.uuid.empty() ) + { + elog ("%s refusing to add null instance to host\n", hostname.c_str()); + return (FAIL_INVALID_UUID); + } + + /** + * Loop over all hosts looking for this instance. If it exhists for a different host + * then remove it from that host and add it to the specified host. + * This is done because the add is also acting as a modify operation + **/ + for ( struct guest_host * ptr = guest_head ; guest_head ; ptr = ptr->next ) + { + if ( ptr->instance_list.size() != 0 ) + { + for ( ptr->instance_list_ptr = ptr->instance_list.begin(); + ptr->instance_list_ptr != ptr->instance_list.end(); + ptr->instance_list_ptr++ ) + { + if ( !ptr->instance_list_ptr->uuid.compare(instance.uuid)) + { + /* Verify that this instance is for the specified host. + * If not then delete this instance from this host and + * allow it to be added below for the specified host */ + if ( ptr->hostname.compare(hostname) ) + { + /* not this host so delete it from the list */ + ilog ("%s %s move to %s\n", ptr->hostname.c_str(), + ptr->instance_list_ptr->uuid.c_str(), + hostname.c_str() ); + + /* remove the instance from this host's guestServer */ + send_cmd_to_guestServer ( ptr->hostname, + MTC_CMD_DEL_INST, + instance.uuid, + instance.heartbeat.reporting ); + + /* remove it from this hosts list */ + ptr->instance_list.erase(ptr->instance_list_ptr); + } + else + { + rc = PASS ; + + /* TODO: OBSOLETE check + * Update the instance if it is different from what was passed in */ + if ( ptr->instance_list_ptr->uuid.compare(instance.uuid) ) + { + ptr->instance_list_ptr->hostname = hostname ; + ptr->instance_list_ptr->uuid = instance.uuid ; + ptr->instance_list_ptr->heartbeat.reporting = instance.heartbeat.reporting ; + ptr->instance_list_ptr->heartbeat.provisioned = instance.heartbeat.provisioned; + ptr->instance_list_ptr->heartbeat.failures = 0 ; + ptr->instance_list_ptr->heartbeat.b2b_misses = 0 ; + + ptr->instance_list_ptr->restart_to_str = "0" ; + ptr->instance_list_ptr->resume_to_str = "0" ; + ptr->instance_list_ptr->suspend_to_str = "0" ; + ptr->instance_list_ptr->shutdown_to_str = "0" ; + ptr->instance_list_ptr->vote_to_str = "0" ; + + ilog ("%s %s updated info\n", hostname.c_str(), instance.uuid.c_str()); + + /* Setup the new channel */ + send_cmd_to_guestServer ( hostname, + MTC_CMD_ADD_INST, + instance.uuid, + instance.heartbeat.reporting ); + } + else + { + ilog ("%s %s info unchanged\n", + hostname.c_str(), instance.uuid.c_str()); + } + } + break ; + } + } + } + if (( ptr->next == NULL ) || ( ptr == guest_tail )) + break ; + } + + /* If the instance is not found then we need to add it to the specified host */ + if ( rc == FAIL_NOT_FOUND ) + { + struct guestHostClass::guest_host * host_ptr = static_cast(NULL); + host_ptr = guestHostClass::getHost(hostname); + if ( host_ptr ) + { + instance.hostname = hostname ; + + instance.restart_to_str = "0" ; + instance.resume_to_str = "0" ; + instance.suspend_to_str = "0" ; + instance.shutdown_to_str = "0" ; + instance.vote_to_str = "0" ; + + instance.heartbeat.provisioned = true ; + instance.heartbeat.failures = 0 ; + instance.heartbeat.b2b_misses = 0 ; + host_ptr->instance_list.push_back (instance); + + ilog ("%s %s add - Prov: %s Notify: %s\n", + hostname.c_str(), + instance.uuid.c_str(), + instance.heartbeat.provisioned ? "YES" : "no ", + instance.heartbeat.reporting ? "YES" : "no " ); + + send_cmd_to_guestServer ( hostname, MTC_CMD_ADD_INST, instance.uuid, instance.heartbeat.reporting ); + rc = PASS ; + } + else + { + elog ("%s hostname is unknown (%s)\n", hostname.c_str(), instance.uuid.c_str() ); + rc = FAIL_INVALID_HOSTNAME ; + } + } + return (rc); +} + +/***************************************************************************** + * + * Name : mod_inst + * + * Purpose : Modify an instance's services' state(s) + * + *****************************************************************************/ + +int guestHostClass::mod_inst ( string hostname, instInfo & instance ) +{ + int rc = FAIL_NOT_FOUND ; + + if ( instance.uuid.empty() ) + { + elog ("%s empty instance uuid\n", hostname.c_str()); + return (FAIL_INVALID_UUID); + } + + /** + * First search for this instance. + * If it is found against a different instance then we need to delete the + * instance from that host and add it to he new host. + **/ + for ( struct guest_host * ptr = guest_head ; guest_head ; ptr = ptr->next ) + { + if ( ptr->instance_list.size() != 0 ) + { + for ( ptr->instance_list_ptr = ptr->instance_list.begin(); + ptr->instance_list_ptr != ptr->instance_list.end(); + ptr->instance_list_ptr++ ) + { + if ( !ptr->instance_list_ptr->uuid.compare(instance.uuid)) + { + /* Verify that this instance is for the specified host. + * If not then delete this instance from this host and + * allow it to be added below for the specified host */ + if ( !ptr->hostname.compare(hostname) ) + { + /* This instance is provisioned for this host */ + + /* Manage its state */ + if ( ptr->instance_list_ptr->heartbeat.reporting != instance.heartbeat.reporting ) + { + ptr->instance_list_ptr->heartbeat.reporting = + instance.heartbeat.reporting ; + + ilog ("%s %s instance reporting state changed to %s\n", + ptr->hostname.c_str(), ptr->instance_list_ptr->uuid.c_str(), + ptr->instance_list_ptr->heartbeat.reporting ? "Enabled" : "Disabled"); + } + else + { + ilog ("%s %s instance reporting state already %s\n", + ptr->hostname.c_str(), ptr->instance_list_ptr->uuid.c_str(), + ptr->instance_list_ptr->heartbeat.reporting ? "Enabled" : "Disabled"); + } + send_cmd_to_guestServer ( ptr->hostname, + MTC_CMD_MOD_INST, + ptr->instance_list_ptr->uuid, + ptr->instance_list_ptr->heartbeat.reporting ); + return (PASS) ; + } + else + { + ilog ("%s %s move to %s while %s\n", + ptr->hostname.c_str(), + ptr->instance_list_ptr->uuid.c_str(), + hostname.c_str(), + ptr->instance_list_ptr->heartbeat.reporting ? "enabled" : "disabled"); + /** + * The instance must have moved to another host. + * Delete it here and then explicitely add + * it below by keeping rc = FAIL_NOT_FOUND + **/ + send_cmd_to_guestServer ( ptr->hostname, + MTC_CMD_DEL_INST, + instance.uuid, + instance.heartbeat.reporting ); + + ptr->instance_list.erase(ptr->instance_list_ptr); + + /* Go through other hosts just to make it easy to exit + * - acts as a safety net */ + break ; + } + } + } + } + if (( ptr->next == NULL ) || ( ptr == guest_tail )) + break ; + } + + /* If the instance is not found then we need to add it to the specified host */ + if ( rc == FAIL_NOT_FOUND ) + { + struct guestHostClass::guest_host * ptr = guestHostClass::getHost(hostname); + if ( ptr ) + { + instance.hostname = hostname ; + + instance.heartbeat.provisioned = true ; + + /* Don't change the reportinfg state */ + instance.heartbeat.reporting = instance.heartbeat.reporting ; + + instance.heartbeat.failures = 0 ; + instance.heartbeat.b2b_misses = 0 ; + + instance.restart_to_str = "0" ; + instance.resume_to_str = "0" ; + instance.suspend_to_str = "0" ; + instance.shutdown_to_str = "0" ; + instance.vote_to_str = "0" ; + + /* The mod might be straight to enabled state */ + ilog ("%s %s instance reporting state is %s\n", + ptr->hostname.c_str(), + instance.uuid.c_str(), + instance.heartbeat.reporting ? "Enabled" : "Disabled"); + + ptr->instance_list.push_back (instance); + send_cmd_to_guestServer ( hostname, + MTC_CMD_ADD_INST, + instance.uuid, + instance.heartbeat.reporting ); + rc = PASS ; + } + else + { + rc = FAIL_INVALID_HOSTNAME ; + } + } + return (rc); +} + + +/***************************************************************************** + * + * Name : del_inst + * + * Purpose : Delete an instance from the guestAgent guestHostClass database + * + * Description: Search all the hosts for this instance and remove it + * from its instance tracking list. + * + * Also send a delete command to the guestServer so that it + * has the opportunity to do any cleanup actions. + * + *****************************************************************************/ +int guestHostClass::del_inst ( string instance_uuid ) +{ + int rc = FAIL_NOT_FOUND ; + + if ( instance_uuid.empty() ) + { + elog ("supplied instance uuid was null\n"); + return (FAIL_INVALID_UUID); + } + + /** Loop over all hosts looking for this instance. */ + for ( struct guest_host * ptr = guest_head ; guest_head ; ptr = ptr->next ) + { + if ( ptr->instance_list.size() != 0 ) + { + for ( ptr->instance_list_ptr = ptr->instance_list.begin(); + ptr->instance_list_ptr != ptr->instance_list.end(); + ptr->instance_list_ptr++ ) + { + if ( !ptr->instance_list_ptr->uuid.compare(instance_uuid)) + { + ilog ("%s removed instance %s\n", + ptr->hostname.c_str(), + instance_uuid.c_str()); + + send_cmd_to_guestServer ( ptr->hostname, + MTC_CMD_DEL_INST, + instance_uuid, + ptr->instance_list_ptr->heartbeat.reporting ); + + ptr->instance_list.erase(ptr->instance_list_ptr); + + return (PASS) ; + } + else + { + jlog ("%s %s:%s (search)\n", + ptr->hostname.c_str(), + ptr->instance_list_ptr->uuid.c_str(), + instance_uuid.c_str()); + } + } + } + if (( ptr->next == NULL ) || ( ptr == guest_tail )) + break ; + } + wlog ("instance was not found '%s'\n", instance_uuid.c_str()); + return (rc); +} + +/** + * Change the host level fault repoorting state for the specified host. + * + * TODO: Consider sending a MOD_HOST command to the guestServer + * + **/ +int guestHostClass::host_inst ( string hostname, mtc_cmd_enum command ) +{ + int rc = FAIL_NOT_FOUND ; + + struct guestHostClass::guest_host * ptr = static_cast(NULL); + + if ( hostname.empty() ) + { + elog ("no hostname specified\n"); + return (FAIL_STRING_EMPTY); + } + ptr = guestHostClass::getHost(hostname); + if ( ptr ) + { + if ( command == MTC_CMD_ENABLE ) + { + ptr->reporting = true ; + ilog ("%s host level heartbeat reporting is Enabled\n", hostname.c_str()); + send_cmd_to_guestServer ( hostname, MTC_CMD_MOD_HOST, ptr->uuid, true ); + } + else + { + ptr->reporting = false ; + ilog ("%s host level heartbeat reporting is Disabled\n", hostname.c_str()); + send_cmd_to_guestServer ( hostname, MTC_CMD_MOD_HOST, ptr->uuid, false ); + } + rc = PASS ; + } + return (rc); +} + + +instInfo * guestHostClass::get_inst ( string instance_uuid ) +{ + if ( instance_uuid.empty() ) + { + elog ("empty instance uuid\n"); + return (NULL); + } + + /** Loop over all hosts looking for this instance. */ + for ( struct guest_host * ptr = guest_head ; guest_head ; ptr = ptr->next ) + { + if ( ptr->instance_list.size() != 0 ) + { + for ( ptr->instance_list_ptr = ptr->instance_list.begin(); + ptr->instance_list_ptr != ptr->instance_list.end(); + ptr->instance_list_ptr++ ) + { + if ( !ptr->instance_list_ptr->uuid.compare(instance_uuid)) + { + dlog ("%s found instance %s\n", + ptr->hostname.c_str(), + ptr->instance_list_ptr->uuid.c_str()); + + return ( &(*ptr->instance_list_ptr) ); + } + } + } + if (( ptr->next == NULL ) || ( ptr == guest_tail )) + break ; + } + return (NULL); +} + +/** returns he number of instances on this host */ +int guestHostClass::num_instances ( string hostname ) +{ + guest_host * guest_host_ptr = getHost ( hostname ); + if ( guest_host_ptr != NULL ) + { + return ( guest_host_ptr->instance_list.size()); + } + return ( 0); +} + +/****************************************************************************/ +/** Host Class Setter / Getters */ +/****************************************************************************/ + +struct guestHostClass::guest_host * guestHostClass::getHost_timer ( timer_t tid ) +{ + /* check for empty list condition */ + if ( tid != NULL ) + { + for ( struct guest_host * host_ptr = guest_head ; guest_head ; host_ptr = host_ptr->next ) + { + if ( host_ptr->host_audit_timer.tid == tid ) + { + return host_ptr ; + } + if (( host_ptr->next == NULL ) || ( host_ptr == guest_tail )) + break ; + } + } + return static_cast(NULL); +} + +static string null_str = "" ; +string guestHostClass::get_host_name ( string uuid ) +{ + guest_host * guest_host_ptr = getHost ( uuid ); + if ( guest_host_ptr != NULL ) + { + return ( guest_host_ptr->hostname ); + } + return ( null_str ); +} + +string guestHostClass::get_host_uuid ( string hostname ) +{ + guest_host * guest_host_ptr = getHost ( hostname ); + if ( guest_host_ptr != NULL ) + { + return ( guest_host_ptr->uuid ); + } + return ( null_str ); +} + + + + + +string guestHostClass::get_inst_host_name ( string instance_uuid ) +{ + if ( instance_uuid.empty() ) + { + elog ("empty instance uuid\n"); + return (null_str); + } + /** Loop over all hosts looking for this instance. */ + for ( struct guest_host * ptr = guest_head ; guest_head ; ptr = ptr->next ) + { + if ( ptr->instance_list.size() != 0 ) + { + for ( ptr->instance_list_ptr = ptr->instance_list.begin(); + ptr->instance_list_ptr != ptr->instance_list.end(); + ptr->instance_list_ptr++ ) + { + if ( !ptr->instance_list_ptr->uuid.compare(instance_uuid)) + { + dlog ("%s found instance %s\n", + ptr->hostname.c_str(), + ptr->instance_list_ptr->uuid.c_str()); + + return ( ptr->hostname ); + } + } + } + if (( ptr->next == NULL ) || ( ptr == guest_tail )) + break ; + } + return ( null_str ); +} + +/** + * Set and Get a bool that indicates whether we already + * got the host reporting state from the VIM. + * + * The VIM might not be running at the time this daemon + * is started so we need to retry until we get it + **/ +void guestHostClass::set_got_host_state ( string hostname ) +{ + guest_host * guest_host_ptr = getHost ( hostname ); + if ( guest_host_ptr ) + { + guest_host_ptr->got_host_state = true ; + } + else + { + wlog ("%s not found\n", hostname.c_str()); + } +} + +bool guestHostClass::get_got_host_state ( string hostname ) +{ + guest_host * guest_host_ptr = getHost ( hostname ); + if ( guest_host_ptr ) + { + return ( guest_host_ptr->got_host_state ); + } + else + { + wlog ("%s not found\n", hostname.c_str()); + } + return (false); +} + +void guestHostClass::set_got_instances ( string hostname ) +{ + guest_host * guest_host_ptr = getHost ( hostname ); + if ( guest_host_ptr ) + { + guest_host_ptr->got_instances = true ; + } + else + { + wlog ("%s not found\n", hostname.c_str()); + } +} + +bool guestHostClass::get_got_instances ( string hostname ) +{ + guest_host * guest_host_ptr = getHost ( hostname ); + if ( guest_host_ptr ) + { + return ( guest_host_ptr->got_instances ); + } + else + { + wlog ("%s not found\n", hostname.c_str()); + } + return (false); +} + +bool guestHostClass::get_reporting_state ( string hostname ) +{ + guest_host * guest_host_ptr = getHost ( hostname ); + if ( guest_host_ptr ) + { + return ( guest_host_ptr->reporting ); + } + else + { + wlog ("%s not found\n", hostname.c_str()); + } + return ( false ); +} + +int guestHostClass::set_reporting_state( string hostname, bool reporting ) +{ + int rc = PASS ; + guest_host * guest_host_ptr = getHost ( hostname ); + if ( guest_host_ptr ) + { + guest_host_ptr->reporting = reporting ; + } + else + { + wlog ("%s not found\n", hostname.c_str()); + rc = FAIL_NOT_FOUND ; + } + return (rc); +} + + +string guestHostClass::get_host_ip ( string hostname ) +{ + guest_host * guest_host_ptr = getHost ( hostname ); + if ( guest_host_ptr ) + { + return ( guest_host_ptr->ip ); + } + return ( null_str ); +} + +void guestHostClass::set_query_flag ( string hostname ) +{ + guest_host * guest_host_ptr = getHost ( hostname ); + if ( guest_host_ptr ) + { + guest_host_ptr->query_flag = true ; + } +} + +void guestHostClass::clr_query_flag ( string hostname ) +{ + guest_host * guest_host_ptr = getHost ( hostname ); + if ( guest_host_ptr ) + { + guest_host_ptr->query_flag = false ; + } +} + +bool guestHostClass::get_query_flag ( string hostname ) +{ + guest_host * guest_host_ptr = getHost ( hostname ); + if ( guest_host_ptr ) + { + return ( guest_host_ptr->query_flag ); + } + return ( false ); +} + +int guestHostClass::set_inst_state ( string hostname ) +{ + guest_host * guest_host_ptr = getHost ( hostname ); + if ( guest_host_ptr ) + { + for ( guest_host_ptr->instance_list_ptr = guest_host_ptr->instance_list.begin(); + guest_host_ptr->instance_list_ptr != guest_host_ptr->instance_list.end(); + guest_host_ptr->instance_list_ptr++ ) + { + send_cmd_to_guestServer ( hostname, MTC_CMD_MOD_INST, + guest_host_ptr->instance_list_ptr->uuid, + guest_host_ptr->instance_list_ptr->heartbeat.reporting ); + } + } + return (PASS); +} + +void guestHostClass::inc_query_misses ( string hostname ) +{ + guest_host * guest_host_ptr = getHost ( hostname ); + if ( guest_host_ptr ) + { + guest_host_ptr->query_misses++ ; + } + else + { + /* TODO: turn into a wlog_throttled ... */ + dlog ("%s not found\n", hostname.c_str()); + } +} + +void guestHostClass::clr_query_misses ( string hostname ) +{ + guest_host * guest_host_ptr = getHost ( hostname ); + if ( guest_host_ptr ) + { + guest_host_ptr->query_misses = 0 ; + } + else + { + /* TODO: turn into a wlog_throttled ... */ + dlog ("%s not found\n", hostname.c_str()); + } +} + +int guestHostClass::get_query_misses ( string hostname ) +{ + guest_host * guest_host_ptr = getHost ( hostname ); + if ( guest_host_ptr ) + { + return ( guest_host_ptr->query_misses ) ; + } + else + { + /* TODO: turn into a wlog_throttled ... */ + dlog ("%s not found\n", hostname.c_str()); + } + return (-1); +} + +/************************************************************************** + * + * Name : getEvent + * + * Purpose : Return a reference to a host or instance level libEvent. + * + **************************************************************************/ +libEvent & guestHostClass::getEvent ( struct event_base * base_ptr, string & hostname ) +{ + struct guest_host * guest_ptr = static_cast(NULL) ; + + /* check for empty list condition */ + if ( guest_head == NULL ) + return (nullEvent) ; + + if ( base_ptr == NULL ) + return (nullEvent) ; + + /** Loop over all hosts looking for this instance. */ + for ( guest_ptr = guest_head ; guest_ptr != NULL ; guest_ptr = guest_ptr->next ) + { + if ( guest_ptr->vimEvent.base == base_ptr ) + { + dlog2 ("%s Found Event Base Pointer (host) (%p)\n", + guest_ptr->vimEvent.uuid.c_str(), + guest_ptr->vimEvent.base); + + /* Update the reference variable */ + hostname = guest_ptr->hostname ; + + return (guest_ptr->vimEvent) ; + } + else if ( guest_ptr->instance_list.size() ) + { + for ( guest_ptr->instance_list_ptr = guest_ptr->instance_list.begin(); + guest_ptr->instance_list_ptr != guest_ptr->instance_list.end(); + guest_ptr->instance_list_ptr++ ) + { + if ( guest_ptr->instance_list_ptr->vimEvent.base == base_ptr ) + { + dlog2 ("%s Found Event Base Pointer (instance) (%p)\n", + guest_ptr->instance_list_ptr->uuid.c_str(), + guest_ptr->instance_list_ptr->vimEvent.base); + + /* Update the reference variable */ + hostname = guest_ptr->hostname ; + + return (guest_ptr->instance_list_ptr->vimEvent) ; + } + } + } + if (( guest_ptr->next == NULL ) || ( guest_ptr == guest_tail )) + break ; + } + return (nullEvent) ; +} + +libEvent & guestHostClass::get_host_event ( string hostname ) +{ + guestHostClass::guest_host * guest_host_ptr ; + guest_host_ptr = guestHostClass::getHost ( hostname ); + if ( guest_host_ptr ) + { + return ( guest_host_ptr->vimEvent ); + } + else + { + wlog ("%s not found\n", hostname.c_str()); + } + return ( nullEvent ); +} + +/***************************************************************************** + * Memory Dump Stuff * + *****************************************************************************/ +void guestHostClass::print_node_info ( void ) +{ + fflush (stdout); + fflush (stderr); +} + +void guestHostClass::mem_log_info ( void ) +{ + char str[MAX_MEM_LOG_DATA] ; + snprintf (&str[0], MAX_MEM_LOG_DATA, "Hosts:%d Allocs:%d Memory:%d\n", hosts, memory_allocs, memory_used ); + mem_log (str); +} + +void guestHostClass::mem_log_info_host ( struct guestHostClass::guest_host * guest_host_ptr ) +{ + char str[MAX_MEM_LOG_DATA] ; + snprintf (&str[0], MAX_MEM_LOG_DATA, "%s:%s\n", guest_host_ptr->hostname.c_str(), guest_host_ptr->ip.c_str()); + mem_log (str); +} + +void mem_log_delimit_host ( void ) +{ + char str[MAX_MEM_LOG_DATA] ; + snprintf (&str[0], MAX_MEM_LOG_DATA, "-------------------------------------------------------------\n"); + mem_log (str); +} + +void guestHostClass::mem_log_info_inst ( struct guestHostClass::guest_host * ptr ) +{ + char str[MAX_MEM_LOG_DATA] ; + if ( ptr->instance_list.size() ) + { + for ( ptr->instance_list_ptr = ptr->instance_list.begin(); + ptr->instance_list_ptr != ptr->instance_list.end(); + ptr->instance_list_ptr++ ) + { + snprintf (&str[0], MAX_MEM_LOG_DATA, + " %s %s Faults:%d %s %s\n", + ptr->instance_list_ptr->uuid.data(), + ptr->instance_list_ptr->hostname.data(), + ptr->instance_list_ptr->heartbeat.failures, + ptr->instance_list_ptr->heartbeat.provisioned ? "provisioned" : "", + ptr->instance_list_ptr->heartbeat.reporting ? "reporting" : ""); + mem_log (str); + + snprintf (&str[0], MAX_MEM_LOG_DATA, + " Timeouts: Restart:%s Resume:%s Suspend:%s Shutdown:%s Vote:%s\n", + ptr->instance_list_ptr->restart_to_str.data(), + ptr->instance_list_ptr->resume_to_str.data(), + ptr->instance_list_ptr->suspend_to_str.data(), + ptr->instance_list_ptr->shutdown_to_str.data(), + ptr->instance_list_ptr->vote_to_str.data()); + mem_log (str); + + // mem_log_delimit_host (); + } + } + else + { + snprintf (&str[0], MAX_MEM_LOG_DATA, "no instances\n"); + mem_log (str); + } +} + + +void guestHostClass::memDumpNodeState ( string hostname ) +{ + guestHostClass::guest_host* guest_host_ptr ; + guest_host_ptr = guestHostClass::getHost ( hostname ); + if ( guest_host_ptr == NULL ) + { + mem_log ( hostname, ": ", "Not Found in guestHostClass\n" ); + return ; + } + else + { + mem_log_info_host ( guest_host_ptr ); + } +} + +void guestHostClass::memDumpAllState ( void ) +{ + guestHostClass::hostBase.memLogDelimit (); + + mem_log_info ( ); + + /* walk the node list looking for nodes that should be monitored */ + for ( struct guest_host * ptr = guest_head ; ptr != NULL ; ptr = ptr->next ) + { + memDumpNodeState ( ptr->hostname ); + if ( (ptr->hosttype & COMPUTE_TYPE) == COMPUTE_TYPE) + { + mem_log_info_inst ( ptr ); + } + guestHostClass::hostBase.memLogDelimit (); + } +} diff --git a/mtce-guest/src/guestClass.h b/mtce-guest/src/guestClass.h new file mode 100644 index 00000000..2cb3af6d --- /dev/null +++ b/mtce-guest/src/guestClass.h @@ -0,0 +1,202 @@ +#ifndef __INCLUDE_GUESTCLASS_H__ +#define __INCLUDE_GUESTCLASS_H__ + +/* + * Copyright (c) 2015 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + +#include "guestBase.h" +#include "httpUtil.h" /* for ... libEvent and httpUtil_... */ +#include "hostClass.h" + +typedef enum +{ + STAGE__START, + STAGES +} guest_stage_enum ; + + +class guestHostClass +{ + private: + struct guest_host { + + /** Pointer to the previous / next host in the list */ + struct guest_host * prev; + struct guest_host * next; + + string hostname ; + string uuid ; + string ip ; + int hosttype ; + + /** + * Top level gate for the host. + * If false then reporting for all instances are off. + */ + bool reporting; + + bool query_flag ; + int query_misses ; + + /** Instance level Audit timer */ + struct mtc_timer host_audit_timer; + + /** flag that indicates we were able to fetch host state from the VIM */ + bool got_host_state ; + + /** flag that indicates we were able to fetch intances from the VIM */ + bool got_instances ; + + /** Main FSM stage */ + guest_stage_enum stage ; + + /* List of instances for this host */ + list instance_list ; + list::iterator instance_list_ptr; + + libEvent vimEvent ; + }; + + /** List of allocated host memory. + * + * An array of host pointers. + */ + guest_host * host_ptrs[MAX_HOSTS] ; + + /** A memory allocation counter. + * + * Should represent the number of hosts in the linked list. + */ + int memory_allocs ; + + /** A memory used counter + * + * A variable storing the accumulated host memory + */ + int memory_used ; + + + // struct hostBaseClass::host* getHost ( string hostname ); + + struct guest_host * guest_head ; /**< Host Linked List Head pointer */ + struct guest_host * guest_tail ; /**< Host Linked List Tail pointer */ + + struct guestHostClass::guest_host* newHost ( void ); + struct guestHostClass::guest_host* addHost ( string hostname ); + struct guestHostClass::guest_host* getHost ( string hostname ); + int remHost ( string hostname ); + int delHost ( struct guestHostClass::guest_host * guest_host_ptr ); + struct guestHostClass::guest_host* getHost_timer ( timer_t tid ); + + libEvent & getEvent ( struct event_base * base_ptr, string & hostname ); + + const + char * get_guestStage_str ( struct guestHostClass::guest_host * guest_host_ptr ); + int guestStage_change ( struct guestHostClass::guest_host * guest_host_ptr, guest_stage_enum newStage ); + + void mem_log_info ( void ); + void mem_log_info_host ( struct guestHostClass::guest_host * guest_host_ptr ); + void mem_log_info_inst ( struct guestHostClass::guest_host * guest_host_ptr ); + + public: + + guestHostClass(); /**< constructor */ + ~guestHostClass(); /**< destructor */ + + hostBaseClass hostBase ; + + bool exit_fsm ; + void run_fsm ( string hostname ); + + bool audit_run ; + + /** Host level Audit timer */ + struct mtc_timer audit_timer; + + /** This is a list of host names. */ + std::list hostlist ; + std::list::iterator hostlist_iter_ptr ; + + // void guest_fsm ( void ); + + int hosts ; + + /* For select dispatch */ + struct timeval waitd ; + + fd_set inotify_readfds ; + fd_set instance_readfds ; + fd_set message_readfds ; + + int add_host ( string uuid, string address, string hostname, string nodetype ); + int mod_host ( string uuid, string address, string hostname, string nodetype ); + int del_host ( string hostname ); /* delete the host from the daemon - mtcAgent */ + int rem_host ( string hostname ); + + /** Delete all instances for this host */ + int del_host_inst ( string host_uuid ); + + int add_inst ( string hostname, instInfo & instance ); + int mod_inst ( string hostname, instInfo & instance ); + int del_inst ( string instance ); + instInfo * get_inst ( string instance ); + + /* The handler that lib event calls to handle the return response */ + void guestVimApi_handler ( struct evhttp_request *req, void *arg ); + + /** + * Change all the instance service states to enabled or disable + * for the specified host. + **/ + int host_inst( string hostname , mtc_cmd_enum command ); + + /** + * Set and Get a bool that indicates whether we already + * got the host reporting state from the VIM. + * + * The VIM might not be running at the time this daemon + * is started so we need to retry until we get it + **/ + void set_got_host_state ( string hostname ); + bool get_got_host_state ( string hostname ); + void set_got_instances ( string hostname ); + bool get_got_instances ( string hostname ); + + /** returns he number of instances on this host */ + int num_instances ( string hostname ); + + string get_host_name ( string host_uuid ); + string get_host_uuid ( string hostname ); + string get_host_ip ( string hostname ); + string get_inst_host_name ( string instance_uuid ); + + /* Send the instance reporting state to the guestServer on that host + * primarily used to preiodically refresh instance reporting state or + * set it when the guestServer seems to have restarted */ + int set_inst_state ( string hostname ); + + libEvent & get_host_event ( string hostname ); + + void inc_query_misses ( string hostname ); + void clr_query_misses ( string hostname ); + int get_query_misses ( string hostname ); + void set_query_flag ( string hostname ); + void clr_query_flag ( string hostname ); + bool get_query_flag ( string hostname ); + + bool get_reporting_state( string hostname ); + int set_reporting_state( string hostname, bool enabled ); + + void memLogDelimit ( void ); /**< Debug log delimiter */ + void memDumpNodeState ( string hostname ); + void memDumpAllState ( void ); + void print_node_info ( void ); /**< Print node info banner */ +}; + +guestHostClass * get_hostInv_ptr ( void ); + +#endif /* __INCLUDE_GUESTCLASS_H__ */ diff --git a/mtce-guest/src/guestHttpSvr.cpp b/mtce-guest/src/guestHttpSvr.cpp new file mode 100644 index 00000000..d8c651d2 --- /dev/null +++ b/mtce-guest/src/guestHttpSvr.cpp @@ -0,0 +1,1092 @@ +/* + * Copyright (c) 2015-2017 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + + /** @file Wind River Titanium Cloud Guest Daemon's HTTP Server */ + +#ifdef __AREA__ +#undef __AREA__ +#endif +#define __AREA__ "gst" + +#include +#include +#include +#include /* for ... inet_addr , inet_ntoa */ +#include /* for ... LOOPBACK_IP */ +#include /* for ... HTTP_ status definitions */ +#include + +using namespace std; + +#include "daemon_common.h" /* */ + +#include "nodeBase.h" /* Service header */ +#include "nodeTimers.h" /* */ +#include "nodeUtil.h" /* */ +#include "jsonUtil.h" /* for ... jsonUtil_get_key_val */ + +#include "guestUtil.h" /* for ... guestUtil_inst_init */ +#include "guestClass.h" /* */ +#include "guestHttpSvr.h" /* for ... this module */ +#include "guestVimApi.h" /* for ... guestVimApi_inst_action */ + +extern int send_event_to_mtcAgent ( string hostname, unsigned int event); + +/* Used for log messages */ +#define GUEST_SERVER "HTTP Guest Server" + +/** + * HTTP commands level is specified in the URL as either + * of the following ; both are at v1 + **/ +#define HOST_LEVEL_URL "/v1/hosts/" +#define INST_LEVEL_URL "/v1/instances/" + +/* Commands require the "User Agent" to be set to SERVICE_VERSION */ +#define USER_AGENT "User-Agent" +#define SERVICE_VERSION "vim/1.0" + +/* This servers's request structure */ +static request_type guest_request ; + +int sequence = 0 ; +char log_str [MAX_API_LOG_LEN]; +char filename[MAX_FILENAME_LEN]; + +/* Module Cleanup */ +void guestHttpSvr_fini ( void ) +{ + if ( guest_request.fd ) + { + if ( guest_request.base ) + { + event_base_free( guest_request.base); + } + close ( guest_request.fd ); + } +} + +/* Look for events */ +void guestHttpSvr_look ( void ) +{ + /* Look for INV Events */ + if ( guest_request.base ) + event_base_loop( guest_request.base, EVLOOP_NONBLOCK ); +} + +/** + * Formulates and updates the resp_buffer reference + * variable based on the specified error code + **/ +string _create_error_response ( int error_code ) +{ + string resp_buffer = "{" ; + resp_buffer.append (" \"status\" : \"fail\""); + switch (error_code) + { + case FAIL_KEY_VALUE_PARSE: + { + resp_buffer.append (",\"reason\" : \"command parse error\""); + break ; + } + case FAIL_JSON_ZERO_LEN: + { + resp_buffer.append (",\"reason\" : \"no buffer\""); + break ; + } + case FAIL_NOT_FOUND: + { + resp_buffer.append (",\"reason\" : \"entity not found\""); + break ; + + } + case FAIL_INVALID_DATA: + { + resp_buffer.append (",\"reason\" : \"invalid data\""); + break ; + } + case FAIL_BAD_STATE: + { + resp_buffer.append (",\"reason\" : \"bad state\""); + break ; + } + case FAIL_BAD_CASE: + { + resp_buffer.append (",\"reason\" : \"unsupported http command\""); + break ; + } + default: + { + ; + } + } + resp_buffer.append ("}"); + + return (resp_buffer); +} + +/******************************************************************** + * + * Name : _get_service_level + * + * Description: Verify this request contains + * + * 1. valid service level specification in the URL and + * 2. the expected User-Agent value + * + ********************************************************************/ +service_level_enum _get_service_level ( struct evhttp_request *req ) +{ + service_level_enum service_level = SERVICE_LEVEL_NONE ; + + /* Parse Headers we care about to verify that it also contains the correct User-Agent header */ + struct evkeyvalq * headers_ptr = evhttp_request_get_input_headers (req); + const char * header_value_ptr = evhttp_find_header (headers_ptr, USER_AGENT); + if ( header_value_ptr ) + { + if ( strncmp ( header_value_ptr, SERVICE_VERSION, 20 ) ) + { + elog ("Request missing required '%s=%s' (%s)\n", + USER_AGENT, SERVICE_VERSION, header_value_ptr ); + return (service_level); + } + } + + /* get the URL string */ + const char * url_ptr = evhttp_request_get_uri (req); + jlog1 ("URI: %s\n", url_ptr ); + + /* look for the supported service levels in the url */ + const char * service_level_ptr = strstr ( url_ptr, HOST_LEVEL_URL); + if ( service_level_ptr ) + { + service_level = SERVICE_LEVEL_HOST ; + } + else + { + service_level_ptr = strstr ( url_ptr, INST_LEVEL_URL); + if ( service_level_ptr ) + { + service_level = SERVICE_LEVEL_INST ; + } + } + if ( service_level == SERVICE_LEVEL_NONE ) + { + elog ("Unsupported service level (url:%s)\n", url_ptr ); + return (service_level); + } + return (service_level); +} + + + +string _update_services_response ( string hostname, string uuid, instInfo * instinfo_ptr ) +{ + string response = ("{"); + response.append ("\"uuid\":\""); + response.append (uuid); + response.append ("\","); + response.append ("\"hostname\":\""); + response.append (hostname); + response.append ("\","); + response.append ("\"services\": [{ \"service\":\"heartbeat\","); + + response.append ("\"state\":\""); + if ( instinfo_ptr->heartbeat.reporting == true ) + response.append ("enabled"); + else + response.append ("disabled"); + + response.append ("\",\"restart-timeout\":\""); + if ( instinfo_ptr->heartbeating == true ) + { + response.append (instinfo_ptr->restart_to_str); + response.append ("\",\"status\":\""); + response.append ("enabled\"}]}"); + } + else + { + response.append ("0\",\"status\":\""); + response.append ("disabled\"}]}"); + } + return (response); +} + +/***************************************************************************** + * + * Name: guestHttpSvr_vim_req + * + * Handles three 'operations' + * + * 'delete' - based on uuid + * 'modify' - based on list of key - value pairs + * 'add' - based on inventory record + * + ****************************************************************************** + * Test Commands: + * + +Add Instance: +curl -i -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/instances/8d80875b-fa73-4ccb-bce3-1cd4df10449d -d '{"hostname": "compute-1", "uuid" : "8d80875b-fa73-4ccb-bce3-1cd4df10449d", "channel" : "cgts-instance000001", "services" : ["heartbeat"]}' + + + +Disable Instance: heartbeat +curl -i -X PUT -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/instances/8d80875b-fa73-4ccb-bce3-1cd4df10449d -d '{"hostname": "compute-1", "uuid" : "8d80875b-fa73-4ccb-bce3-1cd4df10449d", "channel" : "cgts-instance000001", "services" : [{"service":"heartbeat" , "state":"disabled"}]}' + +Delete Host: +curl -i -X DELETE -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/hosts/8aee436e-d564-459e-a0d8-26c44792a9df + +Enable Host: heartbeat +curl -i -X PUT -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/hosts/8aee436e-d564-459e-a0d8-26c44792a9df/enable -d '{"hostname": "compute-1", "uuid" : "8d80875b-fa73-4ccb-bce3-1cd4df10449d"}' + +Enable Host: heartbeat +curl -i -X GET -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/instances/8d80875b-fa73-4ccb-bce3-1cd4df10449d + +*/ + +/********************************************************************************* + * + * Name : guestHttpSvr_host_req + * + * Description : Handles host level VIM requests + * + ********************************************************************************/ +string guestHttpSvr_host_req ( char * buffer_ptr, + mtc_cmd_enum command, + evhttp_cmd_type http_cmd, + int & http_status_code ) +{ + string response = "" ; + string hostname = "" ; + + int rc = jsonUtil_get_key_val ( buffer_ptr, MTC_JSON_INV_NAME, hostname ); + + if ( rc ) + { + wlog ("Failed to parse command key values (%d)\n", rc ); + ilog ("... %s\n", buffer_ptr ); + + response = _create_error_response ( FAIL_KEY_VALUE_PARSE ); + http_status_code = HTTP_BADREQUEST ; + } + else + { + guestHostClass * obj_ptr = get_hostInv_ptr (); + + string instance_info = "" ; + string instance_uuid = "" ; + string instance_chan = "" ; + + /* WARNING: We only support a single list element for now */ + list services_list ; + services_list.clear() ; + + switch ( http_cmd ) + { + case EVHTTP_REQ_PUT: + { + qlog ("%s VIM CMD: Enable Host\n", hostname.c_str()); + + rc = obj_ptr->host_inst ( hostname, command ); + if ( rc ) + { + elog ("%s Host Enable Request (vim) - Host Not Found\n", hostname.c_str()); + response = _create_error_response ( FAIL_NOT_FOUND ); + http_status_code = HTTP_NOTFOUND ; + + /* Ask mtce for an inventory update */ + send_event_to_mtcAgent ( obj_ptr->hostBase.my_hostname, MTC_EVENT_MONITOR_READY ) ; + + } + else + { + http_status_code = HTTP_OK ; + response = " { \"status\" : \"pass\" }" ; + } + break ; + } + default: + { + wlog ("%s Unsupported http command '%s'\n", + hostname.c_str(), getHttpCmdType_str(http_cmd)); + response = _create_error_response ( FAIL_BAD_CASE ); + http_status_code = HTTP_BADREQUEST ; + } + } + } + return (response); +} + +/********************************************************************************* + * + * Name : _get_key_val + * + * Description : Get valid value from http message and generate error if failed + * + ********************************************************************************/ +int _get_key_val ( char * buffer_ptr, + string key, + string & value, + int & http_status_code, + string & response ) +{ + int rc = jsonUtil_get_key_val ( buffer_ptr, key, value ); + + if ( rc ) + { + wlog ("Failed to extract %s from message\n", key.c_str()); + http_status_code = HTTP_BADREQUEST ; + response = _create_error_response ( FAIL_KEY_VALUE_PARSE ); + } + return rc; +} + +/********************************************************************************* + * + * Name : _get_list + * + * Description : Get valid list from http message and generate error if failed + * + ********************************************************************************/ +int _get_list ( char * buffer_ptr, + string key, + list & list, + int & http_status_code, + string & response ) +{ + int rc = jsonUtil_get_list ( buffer_ptr, key, list ); + + if ( rc ) + { + wlog ("Failed to extract %s from message\n", key.c_str()); + http_status_code = HTTP_BADREQUEST ; + response = _create_error_response ( FAIL_KEY_VALUE_PARSE ); + } + return rc; +} + +#define EVENT_VOTE "vote" +#define EVENT_STOP "stop" +#define EVENT_REBOOT "reboot" +#define EVENT_PAUSE "pause" +#define EVENT_UNPAUSE "unpause" +#define EVENT_SUSPEND "suspend" +#define EVENT_RESUME "resume" +#define EVENT_LIVE_MIGRATE_BEGIN "live_migrate_begin" +#define EVENT_LIVE_MIGRATE_END "live_migrate_end" +#define EVENT_COLD_MIGRATE_BEGIN "cold_migrate_begin" +#define EVENT_COLD_MIGRATE_END "cold_migrate_end" + +string _get_action_timeout ( instInfo * instInfo_ptr, string action ) +{ + if ( instInfo_ptr->heartbeating == false ) + { + ilog ("%s returning timeout of zero while not heartbeating for action '%s'\n", + log_prefix(instInfo_ptr).c_str(), action.c_str()); + return ("0"); + } + if ( !action.compare (EVENT_VOTE) ) + return (instInfo_ptr->vote_to_str); + + if ( !action.compare (EVENT_STOP) ) + return (instInfo_ptr->shutdown_to_str); + if ( !action.compare (EVENT_REBOOT) ) + return (instInfo_ptr->shutdown_to_str); + if ( !action.compare (EVENT_PAUSE) ) + return (instInfo_ptr->suspend_to_str); + if ( !action.compare (EVENT_UNPAUSE) ) + return (instInfo_ptr->resume_to_str); + if ( !action.compare (EVENT_SUSPEND) ) + return (instInfo_ptr->suspend_to_str); + if ( !action.compare (EVENT_RESUME) ) + return (instInfo_ptr->resume_to_str); + + if ( !action.compare (EVENT_LIVE_MIGRATE_BEGIN) ) + return (instInfo_ptr->suspend_to_str); + if ( !action.compare (EVENT_LIVE_MIGRATE_END) ) + return (instInfo_ptr->resume_to_str); + if ( !action.compare (EVENT_COLD_MIGRATE_BEGIN) ) + return (instInfo_ptr->suspend_to_str); + if ( !action.compare (EVENT_COLD_MIGRATE_END) ) + return (instInfo_ptr->resume_to_str); + + ilog ("%s returning timeout of zero for invalid action '%s'\n", + log_prefix(instInfo_ptr).c_str(), action.c_str()); + + return ("0"); +} + +/********************************************************************************* + * + * Name : guestHttpSvr_inst_req + * + * Description : Handles instance level VIM requests + * + ********************************************************************************/ +string guestHttpSvr_inst_req ( char * buffer_ptr, + mtc_cmd_enum command, + evhttp_cmd_type http_cmd, + int & http_status_code ) +{ + string response = "" ; + string hostname = "" ; + string instance_uuid = "" ; + + _get_key_val ( buffer_ptr, MTC_JSON_INV_NAME, hostname, http_status_code, response); + if ( _get_key_val ( buffer_ptr, MTC_JSON_INV_NAME, hostname, http_status_code, response )) + return (response); + + if ( _get_key_val ( buffer_ptr, "uuid", instance_uuid, http_status_code, response )) + return (response); + + instInfo instance_info ; guestUtil_inst_init ( &instance_info ); + instance_info.uuid = instance_uuid; + + guestHostClass * obj_ptr = get_hostInv_ptr (); + + /* WARNING: We only support a single list element for now */ + list services_list ; + services_list.clear() ; + + switch ( http_cmd ) + { + case EVHTTP_REQ_POST: + { + if ( MTC_CMD_VOTE == command ) + { + jlog ("vote instance Info: %s", buffer_ptr ); + + string action = ""; + if ( _get_key_val (buffer_ptr, "action", action, http_status_code, response ) ) + return (response); + + qlog ("VIM CMD: Vote instance %s\n", + instance_info.uuid.c_str()); + + instInfo * instInfo_ptr = obj_ptr->get_inst ( instance_uuid ); + if ( instInfo_ptr ) + { + response = ("{\"uuid\":\""); + response.append (instance_uuid); + response.append ("\",\"hostname\":\""); + response.append (hostname); + response.append ("\",\"action\":\""); + response.append (action); + response.append ("\",\"timeout\":\""); + response.append (_get_action_timeout ( instInfo_ptr, EVENT_VOTE )); + response.append ("\"}"); + + jlog ("%s %s Vote Response: %s\n", hostname.c_str(), + instance_uuid.c_str(), + response.c_str()); + + + if ( instInfo_ptr->heartbeating ) + send_cmd_to_guestServer (hostname, MTC_CMD_VOTE_INST, instance_uuid, true, action); + } + else + { + elog ("%s %s vote request (from vim) - Instance Not Found\n", hostname.c_str(), instance_uuid.c_str()); + response = _create_error_response ( FAIL_NOT_FOUND ); + http_status_code = HTTP_NOTFOUND ; + } + } + else if ( MTC_CMD_NOTIFY == command ) + { + jlog ("notify instance Info: %s", buffer_ptr ); + + string action = ""; + if ( _get_key_val (buffer_ptr, "action", action, http_status_code, response )) + return (response); + + qlog ("%s %s VIM CMD: Notify instance\n", + hostname.c_str(), instance_info.uuid.c_str()); + + instInfo * instInfo_ptr = obj_ptr->get_inst ( instance_uuid ); + if ( instInfo_ptr ) + { + response = ("{\"uuid\":\""); + response.append (instance_uuid); + response.append ("\",\"hostname\":\""); + response.append (hostname); + response.append ("\",\"action\":\""); + response.append (action); + response.append ("\",\"timeout\":\""); + response.append (_get_action_timeout ( instInfo_ptr , action )); + response.append ("\"}"); + + jlog ("%s %s Notify Response: %s\n", hostname.c_str(), instInfo_ptr->uuid.c_str(), response.c_str()); + + if ( instInfo_ptr->heartbeating ) + send_cmd_to_guestServer (hostname, MTC_CMD_NOTIFY_INST, instance_uuid, true, action); + } + else + { + elog ("%s %s notify request (vim) - Instance Not Found\n", hostname.c_str(), instance_uuid.c_str()); + response = _create_error_response ( FAIL_NOT_FOUND ); + http_status_code = HTTP_NOTFOUND ; + } + } + /* Add instance */ + else + { + if ( _get_list (buffer_ptr, "services", services_list, http_status_code, response )) + return (response); + + string service = services_list.front(); + qlog ("%s %s VIM CMD: Add Instance\n", + hostname.c_str(), + instance_info.uuid.c_str()); + + ilog ("%s %s add request (from vim) (%s)\n", + hostname.c_str(), + instance_info.uuid.c_str(), + service.c_str()); + + if ( obj_ptr->add_inst ( hostname , instance_info ) != PASS ) + { + response = _create_error_response ( FAIL_INVALID_DATA ); + http_status_code = HTTP_BADREQUEST ; + } + else + { + instance_info.heartbeat.provisioned = true ; + response = " { \"status\" : \"pass\" }" ; + } + } + break ; + } + /* PATCH is used to control service states ; enable or disable */ + case EVHTTP_REQ_PATCH: + { + if ( _get_list (buffer_ptr, "services", services_list, http_status_code, response ) ) + return (response); + + jlog ("%s modify instance (%s)", hostname.c_str(), buffer_ptr ); + + string service , state ; + string services = services_list.front() ; + jsonUtil_get_key_val ( (char*)services.data(), "service", service ); + jsonUtil_get_key_val ( (char*)services.data(), "state" , state ); + + qlog ("%s %s VIM CMD: Modify Instance\n", + hostname.c_str(), + instance_info.uuid.c_str()); + + if ( service.compare("heartbeat")) + { + response = _create_error_response ( FAIL_INVALID_DATA ); + http_status_code = HTTP_BADREQUEST ; + return (response); + } + else if ( !state.compare("enabled")) + instance_info.heartbeat.reporting = true; + else if ( !state.compare("disabled")) + instance_info.heartbeat.reporting = false; + else + { + elog ("%s modify request (vim) - invalid instance state '%s'\n", hostname.c_str(), state.c_str()); + response = _create_error_response ( FAIL_BAD_STATE ); + http_status_code = HTTP_BADREQUEST ; + return (response); + } + int rc = obj_ptr->mod_inst ( hostname, instance_info ); + if ( rc ) + { + elog ("%s %s modify request (vim) - Instance Not Found\n", + hostname.c_str(), instance_info.uuid.c_str()); + + response = _create_error_response ( FAIL_NOT_FOUND ); + http_status_code = HTTP_NOTFOUND ; + return (response); + } + else + { + instInfo * instInfo_ptr = obj_ptr->get_inst ( instance_info.uuid ); + if ( instInfo_ptr ) + { + response = _update_services_response ( hostname , instInfo_ptr->uuid, instInfo_ptr ); + http_status_code = HTTP_OK ; + } + else + { + response = _create_error_response ( FAIL_NOT_FOUND ); + http_status_code = HTTP_NOTFOUND ; + } + } + break ; + } + default: + { + wlog ("%s Unsupported HTTP '%s' request\n", instance_uuid.c_str(), getHttpCmdType_str(http_cmd)); + } + } + return (response); +} + +/***************************************************************************** + * + * Name: guestHttpSvr_vimCmdHdlr + * + * Description: Receive an http request extract the request type and buffer from + * it and call process request handler. + * Send the processed message response back to the connection. + * + * Supported requests include: POST, PUT, DELETE + * + ******************************************************************************/ + +int _get_url_info ( struct evhttp_request * req, + const char * url_ptr, + url_info_type & url_info ) +{ + size_t len = 0 ; + + /* Extract the service level from the request URL ; host or instance */ + url_info.service_level = _get_service_level ( req ); + if ( url_info.service_level == SERVICE_LEVEL_NONE ) + { + return ( FAIL_INVALID_DATA ); + } + + /* Remove the service level from the URL */ + if ( url_info.service_level == SERVICE_LEVEL_HOST ) + { + url_ptr = strstr ( url_ptr, HOST_LEVEL_URL ); + len = strlen ( HOST_LEVEL_URL ); + } + else + { + url_ptr = strstr ( url_ptr, INST_LEVEL_URL); + len = strlen ( INST_LEVEL_URL ); + } + + if ( url_ptr ) + { + url_ptr += len ; + url_info.temp = url_ptr ; + url_info.uuid = url_info.temp.substr ( 0 , UUID_LEN ); + } + else + { + ilog ("Failed to parse URL (%s)", url_ptr); // DLOG + return (FAIL_INVALID_UUID) ; + } + /** + * Check to see if there is a command enable/disable/etc after the UUID + * ... If what is left on the URL is longer than a UUID then + * there must be a command so lets get that + **/ + if ( url_info.temp.length() > UUID_LEN ) + { + url_info.command = url_info.temp.substr(url_info.uuid.length()+1, string::npos ); + dlog ("Command:%s\n", url_info.command.c_str()); + } + return (PASS); +} + +void guestHttpSvr_vimCmdHdlr (struct evhttp_request *req, void *arg) +{ + int rc ; + struct evbuffer *resp_buf ; + url_info_type url_info ; + + int http_status_code = HTTP_NOTFOUND ; + guestHostClass * obj_ptr = get_hostInv_ptr () ; + string response = _create_error_response ( FAIL_JSON_ZERO_LEN ); + + guest_request.req = req ; + jlog1 ("HTTP Request:%p base:%p Req:%p arg:%p\n", &guest_request, + guest_request.base, + guest_request.req, + arg ); + + /* Get sender must be localhost */ + const char * host_ptr = evhttp_request_get_host (req); + if ( strncmp ( host_ptr , "localhost" , 10 )) + { + wlog ("Message received from unknown host (%s)\n", host_ptr ); + + /* TODO: Fail the request if from unknown host */ + } + + const char * url_ptr = evhttp_request_get_uri (req); + + /* Extract the operation */ + evhttp_cmd_type http_cmd = evhttp_request_get_command (req); + jlog1 ("%s request from '%s'\n", getHttpCmdType_str(http_cmd), host_ptr ); + + /* Log the request */ + snprintf (&filename[0], MAX_FILENAME_LEN, "/var/log/%s_request.log", program_invocation_short_name ); + // snprintf (&log_str[0], MAX_API_LOG_LEN-1, "\n%s [%5d] http request seq: %d with %d request from %s:%s", + snprintf (&log_str[0], MAX_API_LOG_LEN-1, "guest services request '%s' (%d) %s:%s", + getHttpCmdType_str(http_cmd), ++sequence, host_ptr, url_ptr ); + // send_log_message ( "controller-0", &filename[0], &log_str[0] ); + jlog ( "%s", log_str ); + + /* Fill in the url_info struct from the url string in the request */ + rc = _get_url_info ( req, url_ptr, url_info ); + if ( rc ) + { + evhttp_send_error ( req, MTC_HTTP_FORBIDDEN, response.data() ); + return ; + } + + switch ( http_cmd ) + { + case EVHTTP_REQ_DELETE: + { + qlog ("%s VIM CMD: Delete Host or Instance\n", url_info.uuid.c_str()); + if ( url_info.service_level == SERVICE_LEVEL_HOST ) + { + /* Nothing to do at the host level for delete. + * Don't try and do a hostname lookup as it may already have been deleted */ + ilog ("%s delete host services request (vim)\n", url_info.uuid.c_str()); + } + else + { + ilog ("%s delete instance request (vim)\n", url_info.uuid.c_str()); + + rc = obj_ptr->del_inst ( url_info.uuid ); + } + if ( rc ) + { + elog ("%s instance not found\n", url_info.uuid.c_str()); + response = _create_error_response ( FAIL_NOT_FOUND ); + http_status_code = HTTP_NOTFOUND ; + } + else + { + http_status_code = HTTP_NOCONTENT ; + response = "{ \"status\" : \"pass\" }" ; + } + break ; + } + + /** + * GET is handled at this level because + * there is no payload with it. + **/ + case EVHTTP_REQ_GET: + { + if ( url_info.service_level == SERVICE_LEVEL_INST ) + { + instInfo * instance_ptr = obj_ptr->get_inst ( url_info.uuid ); + qlog ("%s VIM CMD: Query Instance - Reporting State\n", url_info.uuid.c_str()); + if ( !instance_ptr ) + { + elog ("%s query instance reporting state (vim) failed - Instance Not Found\n", url_info.uuid.c_str()); + response = _create_error_response ( FAIL_NOT_FOUND ); + http_status_code = HTTP_NOTFOUND ; + } + else + { + string hostname = obj_ptr->get_inst_host_name(url_info.uuid); + response = _update_services_response ( hostname , url_info.uuid, instance_ptr ); + http_status_code = HTTP_OK ; + } + } + /* GET the host level reporting state */ + else if ( url_info.service_level == SERVICE_LEVEL_HOST ) + { + string hostname = obj_ptr->get_host_name(url_info.uuid) ; + qlog ("%s VIM CMD: Query Host - Reporting State\n", hostname.c_str()); + if ( hostname.length() ) + { + + response = ("{"); + response.append ("\"uuid\":\""); + response.append (url_info.uuid); + response.append ("\","); + response.append ("\"hostname\":\""); + response.append (hostname); + response.append ("\","); + response.append ("\"state\":\""); + + if ( obj_ptr->get_reporting_state(hostname) == true ) + response.append ("enabled\"}"); + else + response.append ("disabled\"}"); + + http_status_code = HTTP_OK ; + } + else + { + wlog ("%s query host reporting state (vim) failed - Host Not Found\n", url_info.uuid.c_str()); + response = _create_error_response ( FAIL_NOT_FOUND ); + http_status_code = HTTP_NOTFOUND ; + + /* Ask mtce for an inventory update */ + send_event_to_mtcAgent ( obj_ptr->hostBase.my_hostname, MTC_EVENT_MONITOR_READY ) ; + } + } + else + { + http_status_code = HTTP_NOTFOUND ; + slog ("invalid service level\n"); + } + break ; + } + + case EVHTTP_REQ_PUT: + case EVHTTP_REQ_POST: + case EVHTTP_REQ_PATCH: + { + /* GET the host level reporting state */ + if (( http_cmd == EVHTTP_REQ_POST ) && + ( url_info.service_level == SERVICE_LEVEL_HOST )) + { + string hostname = obj_ptr->get_host_name(url_info.uuid) ; + qlog ("%s VIM CMD: Create Host\n", hostname.c_str()); + if ( hostname.length() ) + { + ilog ("%s create host services (vim)\n", hostname.c_str()); + + http_status_code = HTTP_OK ; + response = " { \"status\" : \"pass\" }" ; + } + else + { + wlog ("%s create host (vim) failed - Host Not Found\n", url_info.uuid.c_str()); + response = _create_error_response ( FAIL_NOT_FOUND ); + http_status_code = HTTP_NOTFOUND ; + + /* Ask mtce for an inventory update */ + send_event_to_mtcAgent ( obj_ptr->hostBase.my_hostname, MTC_EVENT_MONITOR_READY ) ; + } + break ; + } + + /* Otherwise for PUTs and instances ; get the payload */ + struct evbuffer *in_buf = evhttp_request_get_input_buffer ( req ); + if ( in_buf ) + { + size_t len = evbuffer_get_length(in_buf) ; + if ( len ) + { + ev_ssize_t bytes = 0 ; + char * buffer_ptr = (char*)malloc(len+1); + jlog1 ("Buffer @ %p contains %ld bytes\n", &in_buf, len ); + + memset ( buffer_ptr, 0, len+1 ); + bytes = evbuffer_remove(in_buf, buffer_ptr, len ); + + if ( bytes <= 0 ) + { + http_status_code = HTTP_BADREQUEST ; + wlog ("http request with no payload\n"); + } + else + { + http_status_code = HTTP_OK ; + mtc_cmd_enum mtc_cmd; + + jlog("%s\n", buffer_ptr ); + + if (!url_info.command.compare("enable") ) + mtc_cmd = MTC_CMD_ENABLE ; + else if (!url_info.command.compare("disable") ) + mtc_cmd = MTC_CMD_DISABLE ; + else if (!url_info.command.compare("vote") ) + mtc_cmd = MTC_CMD_VOTE ; + else if (!url_info.command.compare("notify") ) + mtc_cmd = MTC_CMD_NOTIFY ; + else + mtc_cmd = MTC_CMD_NOT_SET ; + + if ( url_info.service_level == SERVICE_LEVEL_INST ) + { + response = guestHttpSvr_inst_req ( buffer_ptr, + mtc_cmd, + http_cmd, + http_status_code ); + } + else if ( url_info.service_level == SERVICE_LEVEL_HOST ) + { + response = guestHttpSvr_host_req ( buffer_ptr, + mtc_cmd, + http_cmd, + http_status_code ); + } + else + { + slog ("invalid service level\n"); + } + } + free ( buffer_ptr ); + } + else + { + http_status_code = MTC_HTTP_LENGTH_REQUIRED ; + wlog ("http request has no length\n"); + } + } + else + { + http_status_code = HTTP_BADREQUEST ; + wlog ("Http request has no buffer\n"); + } + break ; + } + default: + { + wlog ("Unknown command (%d)\n", http_cmd ); + http_status_code = HTTP_NOTFOUND ; + } + } + + if (( http_status_code >= HTTP_OK) && (http_status_code <= HTTP_NOCONTENT )) + { + resp_buf = evbuffer_new(); + jlog ("Response: %s\n", response.c_str()); + evbuffer_add_printf (resp_buf, "%s\n", response.data()); + evhttp_send_reply (guest_request.req, http_status_code, "OK", resp_buf ); + evbuffer_free ( resp_buf ); + } + else + { + if ( http_status_code == HTTP_NOTFOUND ) + { + wlog ("%s not found\n", url_ptr ); + } + else + { + elog ("HTTP request error:%d ; cmd:%s url:%s\n", + http_status_code, + getHttpCmdType_str(http_cmd), + url_ptr); + elog ("... response:%s\n", response.c_str()); + } + evhttp_send_error (guest_request.req, http_status_code, response.data() ); + } +} + +/***************************************************************** + * + * Name : guestHttpSvr_bind + * + * Description : Setup the HTTP server socket + * + *****************************************************************/ +int guestHttpSvr_bind ( request_type & request ) +{ + int rc ; + int flags ; + int one = 1; + + request.fd = socket(AF_INET, SOCK_STREAM, 0); + if (request.fd < 0) + { + elog ("HTTP server socket create failed (%d:%m)\n", errno); + return FAIL_SOCKET_CREATE ; + } + + /* make socket reusable */ + rc = setsockopt(request.fd, SOL_SOCKET, SO_REUSEADDR, (char *)&one, sizeof(int)); + + memset(&request.addr, 0, sizeof(struct sockaddr_in)); + request.addr.sin_family = AF_INET; + request.addr.sin_addr.s_addr = inet_addr(LOOPBACK_IP) ; + request.addr.sin_port = htons(request.port); + + /* bind port */ + rc = bind ( request.fd, (struct sockaddr*)&request.addr, sizeof(struct sockaddr_in)); + if (rc < 0) + { + elog ("HTTP bind failure for port %d (%d:%m)\n", request.port, errno ); + return FAIL_SOCKET_BIND ; + } + + /* Listen for requests */ + rc = listen(request.fd, 10 ); + if (rc < 0) + { + elog ("HTTP listen failed (%d:%m)\n", errno ); + return FAIL_SOCKET_LISTEN; + } + + /* make non-blocking */ + flags = fcntl ( request.fd, F_GETFL, 0) ; + if ( flags < 0 || fcntl(request.fd, F_SETFL, flags | O_NONBLOCK) < 0) + { + elog ("HTTP set to non-blocking failed (%d:%m)\n", errno ); + return FAIL_SOCKET_OPTION; + } + + return PASS; +} + +/* Setup the http server */ +int guestHttpSvr_setup ( request_type & request ) +{ + int rc = PASS ; + if ( ( rc = guestHttpSvr_bind ( request )) != PASS ) + { + return rc ; + } + else if (request.fd < 0) + { + wlog ("failed to get http server socket file descriptor\n"); + return RETRY ; + } + + request.base = event_base_new(); + if (request.base == NULL) + { + elog ("failed to get http server request base\n"); + return -1; + } + request.httpd = evhttp_new(request.base); + if (request.httpd == NULL) + { + elog ("failed to get httpd server handle\n"); + return -1; + } + + evhttp_set_allowed_methods (request.httpd, EVENT_METHODS ); + + rc = evhttp_accept_socket(request.httpd, request.fd); + if ( rc == -1) + { + elog ("failed to accept on http server socket\n"); + return -1; + } + evhttp_set_gencb(request.httpd, guestHttpSvr_vimCmdHdlr, NULL); + + return PASS ; +} + +/* initialize the mtce http server */ +int guestHttpSvr_init ( int port ) +{ + int rc = PASS ; + memset ( &guest_request, 0, sizeof(request_type)); + guest_request.port = port ; + + for ( ; ; ) + { + rc = guestHttpSvr_setup ( guest_request ); + if ( rc == RETRY ) + { + wlog ("%s bind failed (%d)\n", GUEST_SERVER, guest_request.fd ); + } + else if ( rc != PASS ) + { + elog ("%s start failed (rc:%d)\n", GUEST_SERVER, rc ); + } + else if ( guest_request.fd > 0 ) + { + ilog ("Listening for 'http command' messages on %s:%d\n", + inet_ntoa(guest_request.addr.sin_addr), guest_request.port ); + rc = PASS ; + break ; + } + if ( rc ) mtcWait_secs (5); + } + return ( rc ) ; +} diff --git a/mtce-guest/src/guestHttpSvr.h b/mtce-guest/src/guestHttpSvr.h new file mode 100644 index 00000000..750d3512 --- /dev/null +++ b/mtce-guest/src/guestHttpSvr.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2013, 2015 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + + /** + * @file + * Wind River CGTS Platform Controller Maintenance Daemon + */ + + +typedef struct +{ + struct sockaddr_in addr ; + struct event_base * base ; + struct evhttp_request * req ; + struct evhttp * httpd ; + int fd ; + int port ; +} request_type ; + +void guestHttpSvr_fini ( void ); +int guestHttpSvr_init ( int port ); +int guestHttpSvr_setup ( request_type & request ); +void guestHttpSvr_look ( void ); diff --git a/mtce-guest/src/guestHttpUtil.cpp b/mtce-guest/src/guestHttpUtil.cpp new file mode 100644 index 00000000..a39ef5b4 --- /dev/null +++ b/mtce-guest/src/guestHttpUtil.cpp @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2013, 2015 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + +/** + * @file + * Wind River CGTS Platform Controller Maintenance HTTP Utilities. + * + * Public Interfaces: + * + */ + +#include +#include +#include + +using namespace std; + +#include "httpUtil.h" /* for ... common http utilities */ +#include "jsonUtil.h" /* for ... Json Utilities */ +#include "nodeUtil.h" /* for ... Node Utilities */ + +#include "guestClass.h" /* for ... maintenance class nodeLinkClass */ +#include "guestHttpUtil.h" /* for ... this module header */ +#include "guestVimApi.h" /* for ... guestVimApi_Handler */ + +/* Module init */ +void guestHttpUtil_init ( void ) +{ + return ; +} + +/* Module close */ +void guestHttpUtil_fini ( void ) +{ + return ; +} + +/* ********************************************************************* + * + * Name : guestHttpUtil_status + * + * Description: Extracts and returns the HTTP execution status + * + * *********************************************************************/ + +int guestHttpUtil_status ( libEvent & event ) +{ + int rc = PASS ; + + if ( !event.req ) + { + elog ("%s Invalid request\n", event.hostname.length() ? event.hostname.c_str() : "unknown" ); + return (FAIL_UNKNOWN_HOSTNAME); + } + event.status = event.http_status = evhttp_request_get_response_code (event.req); + switch (event.status) + { + case HTTP_OK: + case 201: + case 202: + case 203: + case 204: + { + dlog ("%s HTTP_OK (%d)\n", event.hostname.c_str(), event.status ); + event.status = PASS ; + break; + } + /* Authentication error - refresh the token */ + case 401: + { + rc = FAIL_AUTHENTICATION ; + break ; + } + case 0: + { + wlog ("%s Status:0 - failed to connect to '%s:%d'\n", + event.hostname.c_str(), event.ip.c_str(), event.port); + event.status = FAIL_HTTP_ZERO_STATUS ; + rc = FAIL_HTTP_ZERO_STATUS ; + break ; + } + default: + { + dlog3 ("%s Status: %d\n", event.hostname.c_str(), event.status ); + rc = event.status ; + break; + } + } + return (rc); +} + +/* *********************************************************************** + * + * Name : guestHttpUtil_api_req + * + * Description: Makes an HTTP request based on all the info + * in the supplied libEvent. + * + * ************************************************************************/ +int guestHttpUtil_api_req ( libEvent & event ) + +{ + http_headers_type hdrs ; + bool has_payload = false; + int hdr_entry = 0 ; + int rc = FAIL ; + void(*handler)(struct evhttp_request *, void *) = NULL ; + + /* Bind the unlock handler */ + handler = &guestVimApi_Handler; + + /* set the timeout */ + event.timeout = HTTP_VIM_TIMEOUT ; + + /* Check for memory leaks */ + if ( event.base ) + { + slog ("%s http base memory leak avoidance (%p) fixme !!\n", + event.log_prefix.c_str(), event.base ); + // event_base_free(event.base); + } + + /* Allocate the base */ + event.base = event_base_new(); + if ( event.base == NULL ) + { + elog ("%s No Memory for Request\n", event.log_prefix.c_str()); + return ( FAIL_EVENT_BASE ); + } + + /* Establish connection */ + else if ( httpUtil_connect ( event )) + { + return (FAIL_CONNECT); + } + + else if ( httpUtil_request ( event, handler )) + { + return (FAIL_REQUEST_NEW); + } + + jlog ("%s Address : %s\n", event.hostname.c_str(), event.token.url.c_str()); + + if ((( event.type != EVHTTP_REQ_GET ) && ( event.type != EVHTTP_REQ_DELETE )) || + ( event.request == VIM_HOST_STATE_QUERY )) + { + has_payload = true ; + + /* Add payload to the output buffer but only for PUT, POST and PATCH requests */ + if ( httpUtil_payload_add ( event )) + { + return (FAIL_PAYLOAD_ADD); + } + jlog ("%s Payload : %s\n", event.hostname.c_str(), + event.payload.c_str() ); + } + + /* Convert port to a string */ + char port_str[10] ; + sprintf ( port_str, "%d", event.port ); + + /* Build the HTTP Header */ + hdrs.entry[hdr_entry].key = "Host" ; + hdrs.entry[hdr_entry].value = event.ip ; + hdrs.entry[hdr_entry].value.append(":") ; + hdrs.entry[hdr_entry].value.append(port_str); + hdr_entry++; + + if ( has_payload == true ) + { + hdrs.entry[hdr_entry].key = "Content-Length" ; + hdrs.entry[hdr_entry].value = httpUtil_payload_len ( &event ); + hdr_entry++; + } + + hdrs.entry[hdr_entry].key = "User-Agent" ; + hdrs.entry[hdr_entry].value = "guest-agent/1.0" ; + hdr_entry++; + + hdrs.entry[hdr_entry].key = "Content-Type" ; + hdrs.entry[hdr_entry].value = "application/json" ; + hdr_entry++; + + hdrs.entry[hdr_entry].key = "Connection" ; + hdrs.entry[hdr_entry].value = "close" ; + hdr_entry++; + hdrs.entries = hdr_entry ; + + /* Add the headers */ + if ( httpUtil_header_add ( &event, &hdrs )) + { + return (FAIL_HEADER_ADD); + } + + event.address = event.token.url ; + + rc = evhttp_make_request ( event.conn, event.req, event.type, event.token.url.data()); + if ( rc == PASS ) + { + evhttp_connection_set_timeout(event.req->evcon, event.timeout); + + /* Default to retry for both blocking and non-blocking command */ + event.status = RETRY ; + event.log_prefix = event.hostname ; + event.log_prefix.append (" "); + event.log_prefix.append (event.service) ; + event.log_prefix.append (" "); + event.log_prefix.append (event.operation) ; + jlog2 ("%s Requested (blocking) (to:%d)\n", event.log_prefix.c_str(), event.timeout); + + /* Send the message with timeout */ + event_base_dispatch(event.base); + + httpUtil_free_conn ( event ); + httpUtil_free_base ( event ); + + return(event.status) ; + } + elog ("%s Call to 'evhttp_make_request' failed (rc:%d)\n", + event.hostname.c_str(), rc); + + return (FAIL_MAKE_REQUEST); +} diff --git a/mtce-guest/src/guestHttpUtil.h b/mtce-guest/src/guestHttpUtil.h new file mode 100644 index 00000000..9e622e96 --- /dev/null +++ b/mtce-guest/src/guestHttpUtil.h @@ -0,0 +1,32 @@ +#ifndef __INCLUDE_GUESTHTTPUTIL_H__ +#define __INCLUDE_GUESTHTTPUTIL_H__ +/* + * Copyright (c) 2013, 2015 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + + /** + * @file + * Wind River CGTS Platform Controller Maintenance ... + * + * libevent HTTP support utilities and control structure support header + */ + +#include /* for ... string */ +#include /* for ... http libevent client */ + +using namespace std; + +#include "guestClass.h" /* for ... maintenance class nodeLinkClass */ +#include "httpUtil.h" /* for ... common http utilities */ + +/***********************************************************************/ + +void guestHttpUtil_init ( void ); +void guestHttpUtil_fini ( void ); +int guestHttpUtil_status ( libEvent & event ); +int guestHttpUtil_api_req ( libEvent & event ); + +#endif /* __INCLUDE_GUESTHTTPUTIL_H__ */ diff --git a/mtce-guest/src/guestInstClass.cpp b/mtce-guest/src/guestInstClass.cpp new file mode 100644 index 00000000..bffae409 --- /dev/null +++ b/mtce-guest/src/guestInstClass.cpp @@ -0,0 +1,764 @@ +/* + * Copyright (c) 2013-2016 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + + /** + * @file + * Wind River CGTS Platform Guest Services "Instances Base Class" + */ + + +#include +#include +#include +#include +#include +#include + +using namespace std; + +#include "nodeBase.h" /* for ... common definitions */ +#include "nodeEvent.h" /* for ... set_inotify_watch_file */ +#include "nodeTimers.h" /* for ... mtcTimer */ +#include "guestBase.h" /* for ... instInfo */ +#include "guestUtil.h" /* for ... guestUtil_inst_init */ +#include "guestInstClass.h" /* for ... get_inst */ +#include "guestSvrUtil.h" /* for ... hb_get_state_name */ + +/**< constructor */ +guestInstClass::guestInstClass() +{ + inst_head = NULL ; + inst_tail = NULL ; + + memory_allocs = 0 ; + memory_used = 0 ; + instances = 0 ; + + for ( int i = 0 ; i < MAX_INSTANCES ; i++ ) + { + inst_ptrs[i] = NULL ; + } + + fsm_exit = false ; + reporting = true ; + return ; +} + +/**< destructor */ +guestInstClass::~guestInstClass() +{ + inst * inst_ptr = inst_head ; + inst * temp_ptr = inst_ptr ; + while ( inst_ptr != NULL ) + { + temp_ptr = inst_ptr ; + inst_ptr = inst_ptr->next ; + delInst (temp_ptr); + } + if ( memory_used != 0 ) + { + elog ( "Apparent Memory Leak - Allocs:%d and Bytes:%d\n", + memory_allocs, memory_used ); + } + else + { + dlog ( "No Memory Leaks\n\n"); + } + return ; +} + +void guestInstClass::guest_fsm_run ( void ) +{ + fsm_run (); +} + +/* + * Allocate new instance and tack it on the end of the instance list + */ +struct guestInstClass::inst* guestInstClass::addInst ( string uuid ) +{ + if ( uuid.length() != UUID_LEN ) + { + elog ("invalid instance uuid ; cannot add %s\n", uuid.c_str()); + return static_cast(NULL); + } + + /* verify instance is not already provisioned */ + struct inst * inst_ptr = guestInstClass::getInst ( uuid ); + if ( inst_ptr ) + { + if ( guestInstClass::remInst ( uuid ) ) + { + /* Should never get here but if we do then */ + /* something is seriously wrong */ + elog ("%s unable to remove instance during reprovision\n", + log_prefix(&inst_ptr->instance).c_str()); + return static_cast(NULL); + } + } + + /* allocate memory for new instance */ + inst_ptr = guestInstClass::newInst (); + if( inst_ptr == NULL ) + { + elog ( "failed to allocate memory for new instance\n" ); + return static_cast(NULL); + } + + guestUtil_inst_init ( &inst_ptr->instance ); + + /* Init the new instance */ + inst_ptr->instance.uuid = uuid ; + inst_ptr->query_flag = false ; + inst_ptr->instance.connect_wait_in_secs = DEFAULT_CONNECT_WAIT ; + + /* Init instance's connect and monitor timers */ + /* Assign the timer the instance's name */ + mtcTimer_init ( inst_ptr->reconnect_timer, uuid ); + mtcTimer_init ( inst_ptr->connect_timer, uuid ); + mtcTimer_init ( inst_ptr->monitor_timer, uuid ); + mtcTimer_init ( inst_ptr->init_timer, uuid ); + mtcTimer_init ( inst_ptr->vote_timer, uuid ); + + inst_ptr->action = FSM_ACTION__NONE ; + + inst_ptr->connectStage = INST_CONNECT__START ; + inst_ptr->monitorStage = INST_MONITOR__STEADY ; + inst_ptr->messageStage = INST_MESSAGE__RECEIVE ; + + /* If the instance list is empty add it to the head */ + if( inst_head == NULL ) + { + inst_head = inst_ptr ; + inst_tail = inst_ptr ; + inst_ptr->prev = NULL ; + inst_ptr->next = NULL ; + } + else + { + /* link the new_instance to the tail of the inst_list + * then mark the next field as the end of the inst_list + * adjust tail to point to the last instance + */ + inst_tail->next = inst_ptr ; + inst_ptr->prev = inst_tail ; + inst_ptr->next = NULL ; + inst_tail = inst_ptr ; + } + + instances++ ; + ilog ("%s added as instance %d\n", log_prefix(&inst_ptr->instance).c_str(), instances); + return inst_ptr ; +} + +/* Remove an instance from the linked list of instances - may require splice action */ +int guestInstClass::remInst( string uuid ) +{ + if ( uuid.empty() ) + return -ENODEV ; + + if ( inst_head == NULL ) + return -ENXIO ; + + struct inst * inst_ptr = getInst ( uuid ); + + if ( inst_ptr == NULL ) + return -EFAULT ; + + stop_instance_timers ( inst_ptr ); + + /* Close the channel if it is open */ + guestUtil_close_channel ( &inst_ptr->instance ); + + /* If the instance is the head instance */ + if ( inst_ptr == inst_head ) + { + /* only one instance in the list case */ + if ( inst_head == inst_tail ) + { + dlog2 ("Single Inst -> Head Case\n"); + inst_head = NULL ; + inst_tail = NULL ; + } + else + { + dlog2 ("Multiple Insts -> Head Case\n"); + inst_head = inst_head->next ; + inst_head->prev = NULL ; + } + } + /* if not head but tail then there must be more than one + * instance in the list so go ahead and chop the tail. + */ + else if ( inst_ptr == inst_tail ) + { + dlog2 ("Multiple Inst -> Tail Case\n"); + inst_tail = inst_tail->prev ; + inst_tail->next = NULL ; + } + else + { + dlog2 ("Multiple Inst -> Full Splice Out\n"); + inst_ptr->prev->next = inst_ptr->next ; + inst_ptr->next->prev = inst_ptr->prev ; + } + delInst ( inst_ptr ); + instances-- ; + + if ( instances == 0 ) + ilog ("no instances to monitor\n"); + + return (PASS) ; +} + +/* Perform a linked list search for the instance matching the instance name */ +struct guestInstClass::inst* guestInstClass::getInst ( string chan_or_uuid ) +{ + struct inst * inst_ptr = static_cast(NULL) ; + + /* check for empty list condition */ + if ( inst_head ) + { + for ( inst_ptr = inst_head ; inst_ptr != NULL ; inst_ptr = inst_ptr->next ) + { + if ( !inst_ptr->instance.uuid.compare (chan_or_uuid) ) + { + return inst_ptr ; + } + if ( !inst_ptr->instance.chan.compare (chan_or_uuid) ) + { + return inst_ptr ; + } + + if (( inst_ptr->next == NULL ) || ( inst_ptr == inst_tail )) + break ; + } + } + return static_cast(NULL); +} + +/* + * Allocates memory for a new instance and stores its address in inst_ptrs + * + * @param void + * @return pointer to the newly allocted instance memory + */ +struct guestInstClass::inst * guestInstClass::newInst ( void ) +{ + struct guestInstClass::inst * temp_inst_ptr = NULL ; + + if ( memory_allocs == 0 ) + { + memset ( inst_ptrs, 0 , sizeof(struct inst *)*MAX_INSTANCES); + } + + // find an empty spot + for ( int i = 0 ; i < MAX_INSTANCES ; i++ ) + { + if ( inst_ptrs[i] == NULL ) + { + inst_ptrs[i] = temp_inst_ptr = new inst ; + memory_allocs++ ; + memory_used += sizeof (struct guestInstClass::inst); + + return temp_inst_ptr ; + } + } + elog ( "failed to store new instance pointer address\n" ); + return temp_inst_ptr ; +} + +/* Frees the memory of a pre-allocated instance and removes + * it from the inst_ptrs list. + * + * @param instance * pointer to the instance memory address to be freed + * @return int return code { PASS or -EINVAL } + */ +int guestInstClass::delInst ( struct guestInstClass::inst * inst_ptr ) +{ + if ( memory_allocs > 0 ) + { + for ( int i = 0 ; i < MAX_INSTANCES ; i++ ) + { + if ( inst_ptrs[i] == inst_ptr ) + { + delete inst_ptr ; + inst_ptrs[i] = NULL ; + memory_allocs-- ; + memory_used -= sizeof (struct guestInstClass::inst); + return PASS ; + } + } + elog ( "unable to validate memory address being freed\n" ); + } + else + elog ( "free memory called when there is no memory to free\n" ); + + return -EINVAL ; +} + +/*************************************************************************************** + * P U B L I C I N T E R F A C E S + **************************************************************************************/ + +/* Add an instance based on its uuid. + * If the instance already exists then update its info */ +int guestInstClass::add_inst ( string uuid , instInfo & instance ) +{ + int rc = FAIL ; + + struct guestInstClass::inst * inst_ptr = getInst(uuid); + if ( inst_ptr ) + { + ilog ("********************************************************\n"); + ilog ("%s Already provisioned - TODO: Create copy constructor \n", uuid.c_str()); + ilog ("********************************************************\n"); + + /* Send back a retry in case the add needs to be converted to a modify */ + rc = PASS ; + } + /* Otherwise add it as a new instance */ + else + { + if ( uuid.length() != UUID_LEN ) + { + elog ("invalid uuid %s\n", uuid.c_str()); + return (FAIL_INVALID_UUID); + } + + inst_ptr = guestInstClass::addInst(uuid); + if ( inst_ptr ) + { + rc = PASS ; + } + else + { + elog ("failed to add instance '%s'\n", uuid.c_str()); + rc = FAIL_NULL_POINTER ; + } + } + + if ( rc == PASS ) + { + inst_ptr->heartbeat_count = 0 ; + + inst_ptr->mismatch_count = 0 ; + + /* TODO: This needs to be a complete copy - Need copy constructor */ + inst_ptr->instance.heartbeat.failures = 0 ; + inst_ptr->instance.heartbeat.failed = false ; + inst_ptr->instance.heartbeat.reporting = instance.heartbeat.reporting ; + inst_ptr->instance.heartbeat.provisioned = instance.heartbeat.provisioned ; + inst_ptr->instance.heartbeat.state = instance.heartbeat.state ; + inst_ptr->instance.hbState = hbs_server_waiting_init ; + inst_ptr->instance.vnState = hbs_server_waiting_init ; + + inst_ptr->instance.name_log_prefix = "" ; + inst_ptr->instance.uuid_log_prefix = "" ; + + inst_ptr->instance.name = instance.name ; + inst_ptr->instance.inst = instance.inst ; + inst_ptr->instance.connected = instance.connected ; + inst_ptr->instance.heartbeating = instance.heartbeating ; + inst_ptr->instance.chan_fd = instance.chan_fd ; + inst_ptr->instance.chan_ok = instance.chan_ok ; + + inst_ptr->instance.corrective_action = instance.corrective_action ; + inst_ptr->instance.heartbeat_interval_ms = instance.heartbeat_interval_ms ; + + inst_ptr->instance.vote_secs = instance.vote_secs ; + inst_ptr->instance.shutdown_notice_secs = instance.shutdown_notice_secs ; + inst_ptr->instance.suspend_notice_secs = instance.suspend_notice_secs ; + inst_ptr->instance.resume_notice_secs = instance.resume_notice_secs ; + inst_ptr->instance.restart_secs = instance.restart_secs ; + + /* Update the channel */ + if ( instance.chan.length() > UUID_LEN ) + inst_ptr->instance.chan = instance.chan ; + } + return (rc); +} + +/***************************************************************************** + * + * Name : del_inst + * + * Purpose : Delete an instance from the linked list + * + *****************************************************************************/ +int guestInstClass::del_inst ( string uuid ) +{ + int rc = FAIL ; + if ( ! uuid.empty() ) + { + /* free memory */ + rc = remInst ( uuid ); + } + return ( rc ); +} + +/***************************************************************************** + * + * Name : qry_inst + * + * Purpose : Send instance info to the guestAgent + * + *****************************************************************************/ +int guestInstClass::qry_inst ( ) +{ + return ( guestAgent_qry_handler ()); +} + +void guestInstClass::stop_instance_timers ( struct guestInstClass::inst * inst_ptr ) +{ + /* Free the mtc timer if in use */ + if ( inst_ptr->reconnect_timer.tid ) + { + mtcTimer_stop ( inst_ptr->reconnect_timer ); + inst_ptr->reconnect_timer.ring = false ; + inst_ptr->reconnect_timer.tid = NULL ; + } + /* Free the connect timer if in use */ + if ( inst_ptr->connect_timer.tid ) + { + mtcTimer_stop ( inst_ptr->connect_timer ); + inst_ptr->connect_timer.ring = false ; + inst_ptr->connect_timer.tid = NULL ; + } + /* Free the monitor timer if in use */ + if ( inst_ptr->monitor_timer.tid ) + { + mtcTimer_stop ( inst_ptr->monitor_timer ); + inst_ptr->monitor_timer.ring = false ; + inst_ptr->monitor_timer.tid = NULL ; + } + /* Free the init timer if in use */ + if ( inst_ptr->init_timer.tid ) + { + mtcTimer_stop ( inst_ptr->init_timer ); + inst_ptr->init_timer.ring = false ; + inst_ptr->init_timer.tid = NULL ; + } + /* Free the vote timer if in use */ + if ( inst_ptr->vote_timer.tid ) + { + mtcTimer_stop ( inst_ptr->vote_timer ); + inst_ptr->vote_timer.ring = false ; + inst_ptr->vote_timer.tid = NULL ; + } +} + + +void guestInstClass::free_instance_resources ( void ) +{ + /* check for empty list condition */ + if ( inst_head ) + { + for ( struct inst * inst_ptr = inst_head ; ; inst_ptr = inst_ptr->next ) + { + if ( inst_ptr->instance.chan_fd ) + { + ilog ("%s closing fd %d for uuid %s\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.chan_fd, + inst_ptr->instance.uuid.c_str()); + + close ( inst_ptr->instance.chan_fd ); + } + stop_instance_timers ( inst_ptr ); + + if (( inst_ptr->next == NULL ) || ( inst_ptr == inst_tail )) + break ; + } + } +} + + +/****************************************************************************/ +/** FSM Control Utilities */ +/****************************************************************************/ + +void guestInstClass::reconnect_start ( const char * uuid_ptr ) +{ + string uuid = uuid_ptr ; + if ( uuid.length() != UUID_LEN ) + { + elog ("invalid uuid %s (uuid:%ld)\n", uuid.c_str(), uuid.length()); + return ; + } + + struct guestInstClass::inst * inst_ptr = guestInstClass::getInst(uuid); + if ( inst_ptr ) + { + guestUtil_close_channel ( &inst_ptr->instance ); + } + else + { + inst_ptr = guestInstClass::addInst(uuid); + } + + if ( inst_ptr ) + { + instInfo * instInfo_ptr = &inst_ptr->instance ; + if ( instInfo_ptr->fd_namespace.size() ) + { + /* Setup inotify to watch for new instance serial IO channel creations */ + if ( set_inotify_watch_file ( instInfo_ptr->fd_namespace.data(), + instInfo_ptr->inotify_file_fd, + instInfo_ptr->inotify_file_wd)) + { + elog ("%s failed to setup 'inotify' on %s\n", + log_prefix(instInfo_ptr).c_str(), + instInfo_ptr->fd_namespace.c_str()); + } + } + ilog ("%s reconnecting ... %s\n", log_prefix(instInfo_ptr).c_str(), + instInfo_ptr->connected ? " CONNECTED" : "" ); + + if ( inst_ptr->connect_timer.tid ) + mtcTimer_stop ( inst_ptr->connect_timer ); + + inst_ptr->action = FSM_ACTION__CONNECT ; + inst_ptr->connectStage = INST_CONNECT__START ; + + // mtcTimer_start ( inst_ptr->connect_timer, guestTimer_handler, inst_ptr->instance.connect_wait_in_secs ); + + //ilog ("%s connect attempt in %d seconds\n", + // log_prefix(&inst_ptr->instance).c_str(), inst_ptr->instance.connect_wait_in_secs); + instInfo_ptr->connecting = true ; + } + else + { + elog ("%s failed to find or add instance\n", uuid.c_str() ); + } +} + + + +/****************************************************************************/ +/** Inst Class Setter / Getters */ +/****************************************************************************/ + +/***************************************************************************** + * + * Name : get_inst + * + * Purpose : Return a pointer to the instance for a specified uuid + * + *****************************************************************************/ +instInfo * guestInstClass::get_inst ( string uuid ) +{ + struct guestInstClass::inst * inst_ptr = guestInstClass::getInst(uuid); + if ( inst_ptr ) + { + return (&inst_ptr->instance ); + } + return static_cast(NULL); +} + +/***************************************************************************** + * + * Name : getInst_timer + * + * Purpose : Return a pointer to the instance that contains the timer for + * the specified timer ID. + * + *****************************************************************************/ +struct guestInstClass::inst * guestInstClass::getInst_timer ( timer_t tid, int timer_id ) +{ + if ( tid != NULL ) + { + if ( inst_head ) + { + struct inst * inst_ptr ; + for ( inst_ptr = inst_head ; inst_ptr != NULL ; inst_ptr = inst_ptr->next ) + { + if (( timer_id == INST_TIMER_MONITOR ) && (inst_ptr->monitor_timer.tid == tid )) + { + return inst_ptr ; + } + else if (( timer_id == INST_TIMER_CONNECT ) && (inst_ptr->connect_timer.tid == tid )) + { + return inst_ptr ; + } + else if (( timer_id == INST_TIMER_VOTE ) && ( inst_ptr->vote_timer.tid == tid )) + { + return inst_ptr ; + } + else if (( timer_id == INST_TIMER_INIT ) && ( inst_ptr->init_timer.tid == tid )) + { + return inst_ptr ; + } + else if (( timer_id == INST_TIMER_RECONNECT ) && ( inst_ptr->reconnect_timer.tid == tid )) + { + return inst_ptr ; + } + + if (( inst_ptr->next == NULL ) || ( inst_ptr == inst_tail )) + break ; + } + } + } + return static_cast(NULL); +} + +/* Get an instance's heartbeat fault reporting state */ +bool guestInstClass::get_reporting_state ( string uuid ) +{ + guestInstClass::inst * inst_ptr = guestInstClass::getInst ( uuid ); + if ( inst_ptr ) + { + return ( inst_ptr->instance.heartbeat.reporting ); + } + else + { + wlog ("uuid not found '%s'\n", uuid.c_str()); + } + return ( false ); +} + +/* Set an instances heartbeat fault reporting state */ +int guestInstClass::set_reporting_state( string uuid, bool reporting ) +{ + guestInstClass::inst * inst_ptr = guestInstClass::getInst ( uuid ); + if ( inst_ptr ) + { + inst_ptr->instance.heartbeat.reporting = reporting ; + } + else + { + wlog ("uuid not found '%s'\n", uuid.c_str()); + return (FAIL_NOT_FOUND) ; + } + return (PASS); +} + + +/***************************************************************************** + * + * Name : print_all_instances + * + * Purpose: Print a summary of the instances that are currently provisioned + * + *****************************************************************************/ +void guestInstClass::print_all_instances ( void ) +{ + bool found = false; + int i = 0 ; + if ( inst_head ) + { + struct inst * inst_ptr ; + for ( inst_ptr = inst_head ; inst_ptr != NULL ; inst_ptr = inst_ptr->next ) + { + ilog ("%2d %s Heartbeat: Notify:%c Failures:%d\n", i, + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.heartbeat.reporting ? 'Y':'n', + inst_ptr->instance.heartbeat.failures); + found = true ; + i++ ; + if (( inst_ptr->next == NULL ) || ( inst_ptr == inst_tail )) + break ; + } + } + + if ( found == false ) + { + ilog ("no instances provisioned\n"); + } +} + +/***************************************************************************** + * + * Name : print_instances (private) + * + *****************************************************************************/ +void guestInstClass::print_instances ( void ) +{ + print_all_instances(); +} + +/***************************************************************************** + * Memory Dump Stuff * + *****************************************************************************/ +void guestInstClass::print_node_info ( void ) +{ + fflush (stdout); + fflush (stderr); +} + +void guestInstClass::mem_log_info ( void ) +{ + char str[MAX_MEM_LOG_DATA] ; + snprintf (&str[0], MAX_MEM_LOG_DATA, "Instances:%d Allocs:%d Memory:%d\n", instances, memory_allocs, memory_used ); + mem_log (str); +} + +void mem_log_delimit_host ( void ) +{ + char str[MAX_MEM_LOG_DATA] ; + snprintf (&str[0], MAX_MEM_LOG_DATA, "-------------------------------------------------------------\n"); + mem_log (str); +} + +void guestInstClass::mem_log_inst_info ( void ) +{ + char str[MAX_MEM_LOG_DATA] ; + + struct inst * inst_ptr = static_cast(NULL) ; + + for ( inst_ptr = inst_head ; inst_ptr != NULL ; inst_ptr = inst_ptr->next ) + { + snprintf (&str[0], MAX_MEM_LOG_DATA, "Name : %s %s (%s)\n", + inst_ptr->instance.name.data(), + inst_ptr->instance.uuid.data(), + inst_ptr->instance.inst.data()); + mem_log (str); + + snprintf (&str[0], MAX_MEM_LOG_DATA, "Action: %8d Connect:%2d Message:%2d Delay:%d secs\n", + inst_ptr->action, + inst_ptr->connectStage, + inst_ptr->messageStage, + inst_ptr->instance.connect_wait_in_secs); + mem_log (str); + + snprintf (&str[0], MAX_MEM_LOG_DATA, "State : Reporting: %c Failures: %d Failed: %c\n", + inst_ptr->instance.heartbeat.reporting ? 'Y' : 'n', + inst_ptr->instance.heartbeat.failures, + inst_ptr->instance.heartbeat.failed ? 'Y' : 'n' ); + mem_log (str); + + snprintf (&str[0], MAX_MEM_LOG_DATA, "Setup : Select :%2d Channel OK: %c hbState:%s vnState:%s\n", + inst_ptr->instance.chan_fd, + inst_ptr->instance.chan_ok ? 'Y' : 'n' , + hb_get_state_name(inst_ptr->instance.hbState), + hb_get_state_name(inst_ptr->instance.vnState)); + mem_log (str); + + snprintf (&str[0], MAX_MEM_LOG_DATA, "Oper : Connected: %c Heartbeating: %c\n", + inst_ptr->instance.connected ? 'Y' : 'n', + inst_ptr->instance.heartbeating ? 'Y' : 'n'); + mem_log (str); + + mem_log_delimit_host(); + + /* exit if this happens to be the last one in the list */ + if (( inst_ptr->next == NULL ) || ( inst_ptr == inst_tail )) + break ; + } + if ( inst_head == NULL ) + { + snprintf (&str[0], MAX_MEM_LOG_DATA, "no instances\n"); + mem_log (str); + } +} + +void guestInstClass::memDumpAllState ( void ) +{ + mem_log_info ( ); + mem_log_delimit_host (); + mem_log_inst_info (); +} diff --git a/mtce-guest/src/guestInstClass.h b/mtce-guest/src/guestInstClass.h new file mode 100644 index 00000000..6b601e07 --- /dev/null +++ b/mtce-guest/src/guestInstClass.h @@ -0,0 +1,230 @@ +#ifndef __INCLUDE_INSTBASECLASS_H__ +#define __INCLUDE_INSTBASECLASS_H__ + +/* + * Copyright (c) 2013-2018 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + + /** + * @file + * Wind River CGTS Platform Guest Services "Instances Base Class Header" + */ + +#include +#include "guestBase.h" /* for ... instInfo */ + +typedef enum +{ + FSM_ACTION__NONE, + FSM_ACTION__CONNECT, + FSM_ACTION__LAST + +} guest_fsmActions_enum ; + +typedef enum +{ + INST_CONNECT__START = 1, + INST_CONNECT__WAIT = 2, + INST_CONNECT__RETRY = 3, + INST_CONNECT__DONE = 4, + INST_CONNECT__STAGES = 5 +} guest_connectStages_enum ; + +typedef enum +{ + INST_MONITOR__STEADY = 0, + INST_MONITOR__DELAY = 2, + INST_MONITOR__FAILURE = 1, +} guest_monitorStages_enum ; + + +typedef enum +{ + INST_MESSAGE__RECEIVE = 0, + INST_MESSAGE__SEND_INIT_ACK = 1, + INST_MESSAGE__RESP_WAIT = 2, /* Waiting for heartbeat challenge response */ + INST_MESSAGE__SEND_WAIT = 3, /* Waiting for period to expire for challenge resend */ + INST_MESSAGE__TRANSMIT = 4, + INST_MESSAGE__STALL = 5 +} guest_messageStages_enum ; + +class guestInstClass +{ + private: + struct inst { + + /** Pointer to the previous / next host in the list */ + struct inst * prev; + struct inst * next; + + /* Instance info */ + instInfo instance ; + + /** + * Top level gate for the host. + * If false then reporting for all instances are off. + */ + // bool reporting; + + bool query_flag ; + + #define INST_TIMER_MONITOR (0) + #define INST_TIMER_CONNECT (1) + #define INST_TIMER_RECONNECT (2) + #define INST_TIMER_INIT (3) + #define INST_TIMER_VOTE (4) + #define INST_TIMER_MAX (5) + + // number of continuous reads for an instance to deal with + // potential message burst + #define INST_MSG_READ_COUNT 5 + + /** General Purpose instance timer */ + // struct mtc_timer timer; + struct mtc_timer vote_timer; + struct mtc_timer init_timer; + struct mtc_timer monitor_timer; + struct mtc_timer connect_timer; + struct mtc_timer reconnect_timer; + + + guest_connectStages_enum connectStage ; + guest_messageStages_enum messageStage ; + guest_monitorStages_enum monitorStage ; + + guest_fsmActions_enum action ; + + int monitor_handler_count ; + int message_handler_count ; + int connect_handler_count ; + int mismatch_count ; + int heartbeat_count ; + + /* Message list for this instance*/ + list message_list ; + }; + + struct inst * inst_head ; /**< Inst Linked List Head pointer */ + struct inst * inst_tail ; /**< Inst Linked List Tail pointer */ + + /** List of allocated host memory. + * + * An array of host pointers. + */ + inst * inst_ptrs[MAX_HOSTS] ; + + /** A memory allocation counter. + * + * Should represent the number of hosts in the linked list. + */ + int memory_allocs ; + + /** A memory used counter + * + * A variable storing the accumulated instance memory + */ + int memory_used ; + + bool fsm_exit ; + void fsm_run ( void ); + + struct guestInstClass::inst* newInst ( void ); + struct guestInstClass::inst* addInst ( string uuid ); + struct guestInstClass::inst* getInst ( string uuid ); + int remInst ( string uuid ); + int delInst ( struct guestInstClass::inst * inst_ptr ); + void readInst ( void ); + + void print_all_instances ( void ); + + void mem_log_inst_info ( void ); + + struct guestInstClass::inst* getInst_timer ( timer_t tid, int timer_id ); + + int message_handler ( struct guestInstClass::inst * inst_ptr ); + int connect_handler ( struct guestInstClass::inst * inst_ptr ); + int monitor_handler ( struct guestInstClass::inst * inst_ptr ); + + void start_monitor_timer ( struct guestInstClass::inst * inst_ptr ); + + /** Thus member function loops over all the insances and sends + * a json string instances: [uuid:state],[uuid:state]... + * back to the guestAgent. */ + int guestAgent_qry_handler ( void ); + + int send_challenge ( struct guestInstClass::inst * inst_ptr ); + + void manage_comm_loss ( void ); + + void mem_log_info ( void ); + void process_msg(json_object *jobj_msg, struct guestInstClass::inst * inst_ptr); + void parser(char *buf, ssize_t len, json_tokener* tok, int newline_found, struct guestInstClass::inst * inst_ptr); + void handle_virtio_serial_msg(char *buf, ssize_t len, json_tokener* tok, struct guestInstClass::inst * inst_ptr); + + public: + + guestInstClass(); /**< constructor */ + ~guestInstClass(); /**< destructor */ + + bool reporting ; + void print_instances ( void ); + + /** handle an expired timer */ + void timer_handler ( int sig, siginfo_t *si, void *uc); + + struct mtc_timer search_timer; + + int instances ; + void guest_fsm_run ( void ); + + int qry_inst ( void ); + int add_inst ( string uuid, instInfo & instance ); + int mod_inst ( string uuid, instInfo & instance ); + int del_inst ( string uuid ); + instInfo * get_inst ( string uuid ); + + ssize_t write_inst ( instInfo * ptr, const char *message, size_t size); + + void reconnect_start ( const char * uuid_ptr ) ; // string uuid ); + + void set_query_flag ( string uuid ); + bool get_query_flag ( string uuid ); + bool get_reporting_state( string uuid ); + int set_reporting_state( string uuid, bool enabled ); + + int send_vote_notify ( string uuid ); + int send_vote_notify_resp ( char * hostname, string uuid, + string notification_type, + string event_type, + string vote_result, + string reject_reason); + + void send_client_msg_nack ( instInfo * instInfo_ptr, + string log_err); + void handle_parse_failure ( struct guestInstClass::inst * inst_ptr, + const char *key, + struct json_object *jobj_msg); + + /* Called on controlle daemon exit */ + void free_instance_resources ( void ); + void stop_instance_timers ( struct guestInstClass::inst * inst_ptr ); + + /* For select dispatch */ + struct timeval waitd ; + + fd_set inotify_readfds ; + fd_set instance_readfds ; + fd_set message_readfds ; + + void memLogDelimit ( void ); /**< Debug log delimiter */ + void memDumpNodeState ( string uuid ); + void memDumpAllState ( void ); + void print_node_info ( void ); /**< Print node info banner */ +}; + +guestInstClass * get_instInv_ptr ( void ); + +#endif /* __INCLUDE_INSTBASECLASS_H__ */ diff --git a/mtce-guest/src/guestServer.cpp b/mtce-guest/src/guestServer.cpp new file mode 100644 index 00000000..60903524 --- /dev/null +++ b/mtce-guest/src/guestServer.cpp @@ -0,0 +1,580 @@ +/* + * Copyright (c) 2013, 2016 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + + /** + * @file + * Wind River CGTS Platform Guest Heartbeat Server Daemon on Compute + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* for hostent */ +#include +#include +#include +#include +#include +#include /* for close and usleep */ +#include /* for realtime scheduling api */ + +using namespace std; + +#include "nodeBase.h" +#include "daemon_ini.h" /* Ini Parser Header */ +#include "daemon_common.h" /* Common definitions and types for daemons */ +#include "daemon_option.h" /* Common options for daemons */ +#include "nodeUtil.h" /* for ... common utilities */ +#include "jsonUtil.h" /* for ... jason utilities */ +#include "nodeTimers.h" /* for ... maintenance timers */ +#include "nodeMacro.h" /* for ... CREATE_NONBLOCK_INET_UDP_RX_SOCKET */ +#include "nodeEvent.h" /* for ... set_inotify_watch, set_inotify_close */ +#include "guestBase.h" +#include "guestUtil.h" /* for ... guestUtil_inst_init */ +#include "guestSvrUtil.h" /* for ... guestUtil_inotify_events */ +#include "guestVirtio.h" /* for ... virtio_channel_connect */ +#include "guestSvrMsg.h" /* for ... send_to_guestAgent */ +#include "guestInstClass.h" + +/* Where to send events */ +string guestAgent_ip = "" ; + +/***************************************************************************** + * + * The daemon primary instance racking object. + * + * This object is a dynamically managed linked list of tracked insances + * + * @see guestInstClass Module control structure in guestInstClass.h + * + *****************************************************************************/ +guestInstClass instInv ; +guestInstClass * get_instInv_ptr ( void ) { return(&instInv); } + + +/* @see guestBase.h Module control structure + * TODO: Consider obsoleting by moving into class */ +ctrl_type ctrl ; +ctrl_type * get_ctrl_ptr ( void ) +{ + return(&ctrl); +} + +void daemon_sigchld_hdlr ( void ) +{ + ; /* dlog("Received SIGCHLD ... no action\n"); */ +} + +/** + * Daemon Configuration Structure - The allocated struct + * @see daemon_common.h for daemon_config_type struct format. + */ +static daemon_config_type guest_config ; +daemon_config_type * daemon_get_cfg_ptr () { return &guest_config ; } + +/* Cleanup exit handler */ +void daemon_exit ( void ) +{ + daemon_dump_info (); + daemon_files_fini (); + + /* Close the messaging sockets */ + if ( ctrl.sock.server_rx_sock ) + delete (ctrl.sock.server_rx_sock); + + if ( ctrl.sock.server_tx_sock ) + delete (ctrl.sock.server_tx_sock); + + if ( ctrl.sock.agent_rx_float_sock ) + delete (ctrl.sock.agent_rx_float_sock); + + if ( ctrl.sock.agent_tx_sock ) + delete (ctrl.sock.agent_tx_sock); + + /* Turn off inotify */ + set_inotify_close ( ctrl.inotify_dir_fd, ctrl.inotify_dir_wd ); + + instInv.free_instance_resources (); + + fflush (stdout); + fflush (stderr); + + exit (0); +} + +/** Client Config mask */ +#define CONFIG_MASK (CONFIG_CLIENT_RX_PORT |\ + CONFIG_AGENT_RX_PORT) + +/* Startup config read */ +static int _config_handler ( void * user, + const char * section, + const char * name, + const char * value) +{ + daemon_config_type* config_ptr = (daemon_config_type*)user; + + if (MATCH("agent", "rx_port")) + { + config_ptr->agent_rx_port = atoi(value); + config_ptr->mask |= CONFIG_AGENT_RX_PORT ; + } + else if (MATCH("client", "rx_port")) + { + config_ptr->client_rx_port = atoi(value); + config_ptr->mask |= CONFIG_CLIENT_RX_PORT ; + } + else if (MATCH("client", "hbs_pulse_period")) + { + config_ptr->hbs_pulse_period = atoi(value); + } + else if (MATCH("client", "hbs_failure_threshold")) + { + config_ptr->hbs_failure_threshold = atoi(value); + } +#ifdef WANT_REPORT_DELAY + else if (MATCH("timeouts", "start_delay")) + { + config_ptr->start_delay = atoi(value); + } +#endif + else + { + return (PASS); + } + return (FAIL); +} + +/* Read the guest.ini file and load agent */ +/* settings into the daemon configuration */ +int daemon_configure ( void ) +{ + int rc = FAIL ; + + /* Read the ini */ + char config_fn[100] ; + guest_config.mask = 0 ; + sprintf ( &config_fn[0], "/etc/mtc/%s.ini", program_invocation_short_name ); + if (ini_parse(config_fn, _config_handler, &guest_config) < 0) + { + elog("Can't load '%s'\n", config_fn ); + return (FAIL_LOAD_INI); + } + + get_debug_options ( config_fn, &guest_config ); + + /* Verify loaded config against an expected mask + * as an ini file fault detection method */ + if ( guest_config.mask != CONFIG_MASK ) + { + elog ("Configuration load failed (%x)\n", + (( -1 ^ guest_config.mask ) & CONFIG_MASK) ); + rc = FAIL_INI_CONFIG ; + } + else + { + guest_config.mgmnt_iface = daemon_get_iface_master ( guest_config.mgmnt_iface ); + ilog("Guest Agent : %s:%d\n", guest_config.mgmnt_iface, guest_config.client_rx_port ); + + // get_iface_macaddr ( guest_config.mgmnt_iface, my_macaddr ); + get_iface_address ( guest_config.mgmnt_iface, ctrl.address, true ); + get_hostname ( &ctrl.hostname[0], MAX_HOST_NAME_SIZE ); + + ilog("Report Thres: %d\n", guest_config.hbs_failure_threshold ); +#ifdef WANT_REPORT_DELAY + ilog("Report Delay: %d sec\n", guest_config.start_delay ); +#endif + ilog("Deflt Period: %d msec\n", guest_config.hbs_pulse_period ); + rc = PASS ; + } + + return (rc); +} + + +/****************************/ +/* Initialization Utilities */ +/****************************/ + +/* Setup UDP messaging to the guestAgent. */ +int _socket_init ( void ) +{ + int rc = PASS ; + + guestAgent_ip = getipbyname ( CONTROLLER ); + ilog ("ControllerIP: %s\n", guestAgent_ip.c_str()); + + /* Read the ports the socket struct */ + ctrl.sock.agent_rx_port = guest_config.agent_rx_port ; + ctrl.sock.server_rx_port = guest_config.client_rx_port ; + + /****************************/ + /* Setup the Receive Socket */ + /****************************/ + ctrl.sock.server_rx_sock = new msgClassRx(ctrl.address.c_str(), guest_config.client_rx_port, IPPROTO_UDP); + rc = ctrl.sock.server_rx_sock->return_status; + if ( rc ) + { + elog ("Failed to setup 'guestAgent' receiver on port %d\n", + ctrl.sock.server_rx_port ); + return (rc) ; + } + ctrl.sock.server_tx_sock = new msgClassTx(guestAgent_ip.c_str(), guest_config.agent_rx_port, IPPROTO_UDP, guest_config.mgmnt_iface); + rc = ctrl.sock.server_tx_sock->return_status; + if ( rc ) + { + elog ("Failed to setup 'guestServer' transmiter\n" ); + return (rc) ; + } + + return (rc); +} + +/* The main heartbeat service loop */ +int daemon_init ( string iface, string nodeType_str ) +{ + int rc = PASS ; + + ctrl.address.clear() ; + ctrl.address_peer.clear(); + ctrl.nodetype = CGTS_NODE_NULL ; + + /* Init the Inotify descriptors */ + ctrl.inotify_dir_fd = 0 ; + ctrl.inotify_dir_wd = 0 ; + + /* clear hostname */ + memset ( &ctrl.hostname[0], 0, MAX_HOST_NAME_SIZE ); + + /* Initialize socket construct and pointer to it */ + memset ( &ctrl.sock, 0, sizeof(ctrl.sock)); + + /* Assign interface to config */ + guest_config.mgmnt_iface = (char*)iface.data() ; + + if ( (rc = daemon_files_init ( )) != PASS ) + { + elog ("Pid, log or other files could not be opened (rc:%d)\n", rc ); + rc = FAIL_FILES_INIT ; + } + + /* convert node type to integer */ + ctrl.nodetype = get_host_function_mask ( nodeType_str ) ; + ilog ("Node Type : %s (%d)\n", nodeType_str.c_str(), ctrl.nodetype ); + + /* Bind signal handlers */ + if ( daemon_signal_init () != PASS ) + { + elog ("daemon_signal_init failed\n"); + return ( FAIL_SIGNAL_INIT ); + } + + /************************************************************************ + * There is no point continuing with init ; i.e. running daemon_configure, + * initializing sockets and trying to query for an ip address until the + * daemon's configuration requirements are met. Here we wait for those + * flag files to be present before continuing. + ************************************************************************ + * Wait for /etc/platform/.initial_config_complete & /var/run/.goenabled */ + daemon_wait_for_file ( CONFIG_COMPLETE_FILE , 0); + daemon_wait_for_file ( GOENABLED_MAIN_READY , 0); + + /* Configure the client */ + if ( (rc = daemon_configure ()) != PASS ) + { + elog ("Daemon service configuration failed (rc:%d)\n", rc ); + rc = FAIL_DAEMON_CONFIG ; + } + + /* Setup the heartbeat service messaging sockets */ + else if ( (rc = _socket_init ( )) != PASS ) + { + elog ("socket initialization failed (rc:%d)\n", rc ); + rc = FAIL_SOCKET_INIT; + } + + /* Ignore this signal */ + signal(SIGPIPE, SIG_IGN); + + return (rc); +} + +/* + { hostname" : "" , + "instances" : + [ + { "channel" : "" , "services" : + [ + { "service":"heartbeat", "admin":"enabled", "oper":"enabled" , "avail":"available" } + ], + "channel: : "" , "services" : + [ + { "service":"heartbeat", "admin":"enabled", "oper":"enabled" , "avail":"available"} + ] + } + ] + } +*/ + + +int select_failure_count = 0 ; + +void guestInstClass::manage_comm_loss ( void ) +{ + int rc ; + + std::list socks ; + socks.clear(); + + waitd.tv_sec = 0; + waitd.tv_usec = GUEST_SOCKET_TO; + + /* Initialize the master fd_set */ + FD_ZERO(&inotify_readfds); + + /* check for empty list condition */ + if ( inst_head ) + { + for ( struct inst * inst_ptr = inst_head ; inst_ptr != NULL ; inst_ptr = inst_ptr->next ) + { + if ( inst_ptr->instance.inotify_file_fd ) + { + //ilog ("adding inotify_fd %d for %s to select list\n", + // inst_ptr->instance.inotify_file_fd, + // inst_ptr->instance.uuid.c_str()); + + socks.push_front ( inst_ptr->instance.inotify_file_fd ); + FD_SET ( inst_ptr->instance.inotify_file_fd, &inotify_readfds); + } + if (( inst_ptr->next == NULL ) || ( inst_ptr == inst_tail )) + break ; + } + + /* if there are no sockets to monitor then just exit */ + if ( socks.empty() ) + return ; + + /* Call select() and wait only up to SOCKET_WAIT */ + socks.sort(); + rc = select( socks.back()+1, &inotify_readfds, NULL, NULL, &waitd); + if (( rc < 0 ) || ( rc == 0 ) || ( rc > (int)socks.size())) + { + /* Check to see if the select call failed. */ + /* ... but filter Interrupt signal */ + if (( rc < 0 ) && ( errno != EINTR )) + { + wlog_throttled ( select_failure_count, 20, + "socket select failed (%d:%m)\n", errno); + } + else if ( rc > (int)socks.size()) + { + wlog_throttled ( select_failure_count, 100, + "Select return exceeds current file descriptors (%ld:%d)\n", + socks.size(), rc ); + } + else + { + select_failure_count = 0 ; + } + } + else + { + wlog ( "inotify channel event\n"); + + for ( struct inst * inst_ptr = inst_head ; inst_ptr != NULL ; inst_ptr = inst_ptr->next ) + { + if ( inst_ptr->instance.inotify_file_fd ) + { + if (FD_ISSET(inst_ptr->instance.inotify_file_fd, &inotify_readfds) ) + { + ilog ("Watch Event on instance %s\n", inst_ptr->instance.uuid.c_str()); + guestUtil_inotify_events (inst_ptr->instance.inotify_file_fd); + } + } + if (( inst_ptr->next == NULL ) || ( inst_ptr == inst_tail )) + break ; + } + } + } +} + + + +#define MAX_LEN 300 +void daemon_service_run ( void ) +{ + int rc = 0 ; + int count = 0 ; + int flush_thld = 0 ; + + string payload = "" ; /* for the ready event */ + + std::list socks ; + + guestUtil_load_channels (); + + /* Setup inotify to watch for new instance serial IO channel creations */ + if ( set_inotify_watch ( QEMU_CHANNEL_DIR, + ctrl.inotify_dir_fd, + ctrl.inotify_dir_wd ) ) + { + elog ("failed to setup inotify on %s\n", QEMU_CHANNEL_DIR ); + } + + socks.clear(); + socks.push_front (ctrl.sock.server_rx_sock->getFD()); + if ( ctrl.inotify_dir_fd ) + socks.push_front (ctrl.inotify_dir_fd); + else + { + elog ("unable to inotify monitor %s\n", QEMU_CHANNEL_DIR ); + + // TODO: consider exiting daemon + } + socks.sort(); + + mtcTimer_init ( ctrl.timer, ctrl.hostname ); + mtcTimer_init ( instInv.search_timer, ctrl.hostname ); + + mtcTimer_start ( ctrl.timer , guestTimer_handler, 2 ); + mtcTimer_start ( instInv.search_timer, guestTimer_handler, SEARCH_AUDIT_TIME ); + + ilog ("Selects: guestAgent:%d qemuDir:%d\n", ctrl.sock.server_rx_sock->getFD(), ctrl.inotify_dir_fd ); + ilog ("-------------------------------------------------------\n"); + + /* Tell the guestAgent that we started or restarted + * so that it can send instance state data */ + payload = "{\"hostname\":\"" ; + payload.append(ctrl.hostname); + payload.append("\"}"); + + /* Run heartbeat service forever or until stop condition */ + for ( ; ; ) + { + instInv.waitd.tv_sec = 0; + instInv.waitd.tv_usec = GUEST_SOCKET_TO; + + /* Initialize the master fd_set */ + FD_ZERO(&instInv.message_readfds); + + FD_SET ( ctrl.sock.server_rx_sock->getFD(), &instInv.message_readfds); + if ( ctrl.inotify_dir_fd ) + { + FD_SET ( ctrl.inotify_dir_fd, &instInv.message_readfds); + } + + rc = select( socks.back()+1, &instInv.message_readfds, NULL, NULL, &instInv.waitd); + if (( rc < 0 ) || ( rc == 0 ) || ( rc > (int)socks.size())) + { + /* Check to see if the select call failed. */ + /* ... but filter Interrupt signal */ + if (( rc < 0 ) && ( errno != EINTR )) + { + wlog_throttled ( count, 20, "socket select failed (%d:%m)\n", errno); + } + else if ( rc > (int)socks.size()) + { + wlog_throttled ( count, 100, "Select return exceeds current file descriptors (%ld:%d)\n", + socks.size(), rc ); + } + else + { + count = 0 ; + } + } + else if (FD_ISSET(ctrl.sock.server_rx_sock->getFD(), &instInv.message_readfds)) + { + /* clean the rx/tx buffer */ + mtc_message_type msg ; + memset ((void*)&msg,0,sizeof(mtc_message_type)); + + int bytes = ctrl.sock.server_rx_sock->read((char*)&msg.hdr[0], sizeof(mtc_message_type)); + ctrl.address_peer = ctrl.sock.server_rx_sock->get_src_str() ; + mlog1 ("Received %d bytes from %s:%d:guestAgent\n", bytes, + ctrl.sock.server_rx_sock->get_src_str(), + ctrl.sock.server_rx_sock->get_dst_addr()->getPort() ); + print_mtc_message (&msg); + + if ( bytes > 0 ) + { + recv_from_guestAgent ( msg.cmd, &msg.buf[0] ); + } + } + + else if (FD_ISSET(ctrl.inotify_dir_fd, &instInv.message_readfds)) + { + dlog ("%s dir change\n", QEMU_CHANNEL_DIR ); + + guestUtil_inotify_events (ctrl.inotify_dir_fd); + } + + fflush (stdout); + fflush (stderr); + + instInv.guest_fsm_run ( ); + + if ( ctrl.timer.ring == true ) + { + /* restart the timer and try again if this call returns a RETRY */ + if ( send_to_guestAgent ( MTC_EVENT_MONITOR_READY, payload.data()) == RETRY ) + { + mtcTimer_start ( ctrl.timer, guestTimer_handler, 5 ); + } + ctrl.timer.ring = false ; + } + + daemon_signal_hdlr (); + + /* Support the log flush config option */ + if ( guest_config.flush ) + { + if ( ++flush_thld > guest_config.flush_thld ) + { + flush_thld = 0 ; + fflush (stdout); + fflush (stderr); + } + } + } + daemon_exit (); +} +/* Write the daemon /var/log/.dump */ +void daemon_dump_info ( void ) +{ + daemon_dump_membuf_banner (); + + instInv.print_node_info (); + instInv.memDumpAllState (); + + daemon_dump_membuf(); +} + +const char MY_DATA [100] = { "eieio\n" } ; +const char * daemon_stream_info ( void ) +{ + return (&MY_DATA[0]); +} + +/*************************************************************************** + * * + * Module Test Head * + * * + ***************************************************************************/ + +/** Teat Head Entry */ +int daemon_run_testhead ( void ) +{ + int rc = PASS; + return (rc); +} diff --git a/mtce-guest/src/guestStubs.cpp b/mtce-guest/src/guestStubs.cpp new file mode 100644 index 00000000..5a99dc4d --- /dev/null +++ b/mtce-guest/src/guestStubs.cpp @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2013, 2015 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + + /** + * @file + * Wind River CGTS Platform Nodal Health Check Agent Stubs + */ + +#include + +using namespace std; + +#include "nodeBase.h" +#include "nodeUtil.h" diff --git a/mtce-guest/src/guestSvrFsm.cpp b/mtce-guest/src/guestSvrFsm.cpp new file mode 100644 index 00000000..6add43bb --- /dev/null +++ b/mtce-guest/src/guestSvrFsm.cpp @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2013, 2015 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + +/*************************************************************************** + * + * @file + * Wind River CGTS Platform "Guest Services - Finite State Machine" + * + * + * This FSM handles the following actions + * + * FSM_ACTION__CONNECT + * + */ + +#include +#include + +using namespace std; + +#include "nodeBase.h" +#include "nodeTimers.h" +#include "guestBase.h" +#include "guestInstClass.h" +#include "guestSvrUtil.h" + +void guestInstClass::fsm_run ( void ) +{ + int rc = PASS ; + struct inst * inst_ptr = static_cast(NULL) ; + + if (( instances > 0 ) ) + { + /* get new messages */ + readInst(); + + for ( inst_ptr = inst_head ; inst_ptr != NULL ; inst_ptr = inst_ptr->next ) + { + if ( inst_ptr->message_list.size() ) + { + guestInstClass::message_handler ( inst_ptr ); + } + + if ( inst_ptr->action == FSM_ACTION__NONE ) + { + guestInstClass::monitor_handler ( inst_ptr ); + } + + else if ( inst_ptr->action == FSM_ACTION__CONNECT ) + { + rc = guestInstClass::connect_handler ( inst_ptr ); + if ( rc == RETRY ) + return ; + } + else + { + slog ("unknown action (%d) for instance %s\n", + inst_ptr->action, inst_ptr->instance.uuid.c_str()); + } + +#ifdef WANT_LOSS_FIT + if ( inst_ptr->heartbeat_count > 10 ) + { + mtcTimer_stop ( inst_ptr->monitor_timer ); + mtcWait_secs (1); + start_monitor_timer ( inst_ptr ); + inst_ptr->heartbeat_count = 0 ; + } +#endif + + /* exit if this happens to be the last one in the list */ + if (( inst_ptr->next == NULL ) || ( inst_ptr == inst_tail )) + break ; + } + } + else if ( inst_head != NULL ) + { + slog ("head pointer is not NULL while there are no instances (%p)\n", inst_head ); + } + + if ( search_timer.ring == true ) + { + guestUtil_channel_search (); + mtcTimer_start ( search_timer, guestTimer_handler, SEARCH_AUDIT_TIME ); + } + + /* Make this part of the connect FSM */ + manage_comm_loss ( ); +} diff --git a/mtce-guest/src/guestSvrHdlr.cpp b/mtce-guest/src/guestSvrHdlr.cpp new file mode 100644 index 00000000..c8ab31ca --- /dev/null +++ b/mtce-guest/src/guestSvrHdlr.cpp @@ -0,0 +1,1420 @@ +/* + * Copyright (c) 2015-2018 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + +/**************************************************************************** + * @file + * Wind River CGTS Platform Guest Services "Handlers" Implementation + * + * Description: This file contains the following FSM handlers, + + * Interfaces: + * + * guestInstClass::timer_handler + * guestInstClass::monitor_handler + * guestInstClass::connect_handler + * + ****************************************************************************/ + +#include +#include "nodeBase.h" +#include "nodeUtil.h" /* for ... clean_bm_response_files */ +#include "nodeTimers.h" /* for ... mtcTimer_start/stop */ +#include "jsonUtil.h" /* for ... jsonApi_array_value */ +#include "daemon_common.h" + +#include "guestBase.h" /* for ... */ +#include "guestUtil.h" /* for ... guestUtil_print_instance */ +#include "guestSvrUtil.h" /* for ... hb_get_message_type_name */ +#include "guestVirtio.h" /* for ... */ +#include "guestSvrMsg.h" /* for ... */ +#include "guestInstClass.h" /* for ... */ + +static int failure_reporting_count = 0 ; + +void voteStateChange ( instInfo * instInfo_ptr , hb_state_t newState ) +{ + if ( instInfo_ptr->vnState == newState ) + return ; + + clog ("%s '%s' -> '%s'\n", + log_prefix(instInfo_ptr).c_str(), + hb_get_state_name(instInfo_ptr->vnState), + hb_get_state_name(newState)); + + instInfo_ptr->vnState = newState ; +} + +void beatStateChange ( instInfo * instInfo_ptr , hb_state_t newState ) +{ + if ( instInfo_ptr->hbState == newState ) + return ; + + if ((( instInfo_ptr->hbState == hbs_server_waiting_challenge ) && + ( newState == hbs_server_waiting_response )) || + (( instInfo_ptr->hbState == hbs_server_waiting_response ) && + ( newState == hbs_server_waiting_challenge ))) + { + ; /* don't print heartbeat state changes */ + } + else if (( newState == hbs_server_waiting_init ) && + ( instInfo_ptr->hbState != hbs_server_waiting_init )) + { + ilog ("%s waiting for init ... \n", log_prefix(instInfo_ptr).c_str()); + } + else + { + clog ("%s '%s' -> '%s'\n", + log_prefix(instInfo_ptr).c_str(), + hb_get_state_name(instInfo_ptr->hbState), + hb_get_state_name(newState)); + } + instInfo_ptr->hbState = newState ; +} + +void hbStatusChange ( instInfo * instInfo_ptr, bool status ) +{ + if ( instInfo_ptr->heartbeating != status ) + { + instInfo_ptr->heartbeating = status ; + string payload = guestUtil_set_inst_info ( get_ctrl_ptr()->hostname , instInfo_ptr ); + + if ( status == true ) + { + ilog ("%s is now heartbeating\n", log_prefix(instInfo_ptr).c_str()); + send_to_guestAgent ( MTC_EVENT_HEARTBEAT_RUNNING, payload.data()); + } + else + { + ilog ("%s is not heartbeating\n", log_prefix(instInfo_ptr).c_str()); + send_to_guestAgent ( MTC_EVENT_HEARTBEAT_STOPPED, payload.data()); + } + + jlog ("%s Heartbeating State Change: %s\n", log_prefix(instInfo_ptr).c_str(), payload.c_str()); + } + else + { + clog ("%s heartbeating is still %s\n", + log_prefix(instInfo_ptr).c_str(), status ? "enabled" : "disabled" ); + } +} + + +void manage_heartbeat_failure ( instInfo * instInfo_ptr ) +{ + instInfo_ptr->heartbeat.failed = true ; + + dlog ("%s calling hbStatusChange false\n", log_prefix(instInfo_ptr).c_str()); + + hbStatusChange ( instInfo_ptr, false) ; /* heartbeating is now false */ + + beatStateChange ( instInfo_ptr, hbs_server_waiting_init ) ; +} + +/* Looks up the timer ID and asserts the corresponding node's ringer */ +void guestInstClass::timer_handler ( int sig, siginfo_t *si, void *uc) +{ + struct guestInstClass::inst * inst_ptr ; + timer_t * tid_ptr = (void**)si->si_value.sival_ptr ; + + ctrl_type * ctrl_ptr = get_ctrl_ptr(); + + /* Avoid compiler errors/warnings for parms we must + * have but currently do nothing with */ + sig=sig ; uc = uc ; + + if ( !(*tid_ptr) ) + { + return ; + } + + else if ( *tid_ptr == search_timer.tid ) + { + mtcTimer_stop_int_safe ( search_timer ); + search_timer.ring = true ; + return ; + } + else if ( *tid_ptr == ctrl_ptr->timer.tid ) + { + mtcTimer_stop_int_safe ( ctrl_ptr->timer ); + ctrl_ptr->timer.ring = true ; + return ; + } + + for ( int timer_id = INST_TIMER_MONITOR ; timer_id < INST_TIMER_MAX ; timer_id++ ) + { + if ( ( inst_ptr = guestInstClass::getInst_timer ( *tid_ptr , timer_id ) ) != NULL ) + { + switch ( timer_id ) + { + case INST_TIMER_MONITOR: + { + if (( *tid_ptr == inst_ptr->monitor_timer.tid ) ) + { + mtcTimer_stop_int_safe ( inst_ptr->monitor_timer ); + inst_ptr->monitor_timer.ring = true ; + return ; + } + break ; + } + case INST_TIMER_CONNECT: + { + if (( *tid_ptr == inst_ptr->connect_timer.tid ) ) + { + mtcTimer_stop_int_safe ( inst_ptr->connect_timer ); + inst_ptr->connect_timer.ring = true ; + return ; + } + break ; + } + case INST_TIMER_RECONNECT: + { + if (( *tid_ptr == inst_ptr->reconnect_timer.tid ) ) + { + mtcTimer_stop_int_safe ( inst_ptr->reconnect_timer ); + inst_ptr->reconnect_timer.ring = true ; + + return ; + } + break ; + } + case INST_TIMER_INIT: + { + if (( *tid_ptr == inst_ptr->init_timer.tid ) ) + { + beatStateChange ( &inst_ptr->instance, hbs_server_waiting_init ) ; + mtcTimer_stop_int_safe ( inst_ptr->init_timer ); + return ; + } + break ; + } + case INST_TIMER_VOTE: + { + if (( *tid_ptr == inst_ptr->vote_timer.tid ) ) + { + mtcTimer_stop_int_safe ( inst_ptr->vote_timer ); + inst_ptr->vote_timer.ring = true ; + return ; + } + break ; + } + default: + { + // slog ("unknown timer id (%d)\n", timer_id); + } + } /* end switch */ + } /* end if */ + } /* end for */ +} + +/* guest services timer object wrapper + * - does a instance lookup and calls the timer handler */ +void guestTimer_handler ( int sig, siginfo_t *si, void *uc) +{ + get_instInv_ptr()->timer_handler ( sig, si, uc ); +} + +void guestInstClass::start_monitor_timer ( struct guestInstClass::inst * inst_ptr ) +{ + if ( inst_ptr->monitor_timer.tid ) + mtcTimer_stop ( inst_ptr->monitor_timer ); + + mtcTimer_start_sec_msec ( &inst_ptr->monitor_timer, + guestTimer_handler, + (inst_ptr->instance.heartbeat_interval_ms/1000), + (inst_ptr->instance.heartbeat_interval_ms%1000)); +} + +void _schedule_init_timer ( string event_type , struct mtc_timer & timer ) +{ + if (( !event_type.compare(GUEST_HEARTBEAT_MSG_EVENT_SUSPEND) ) || + ( !event_type.compare(GUEST_HEARTBEAT_MSG_EVENT_LIVE_MIGRATE_BEGIN) ) || + ( !event_type.compare(GUEST_HEARTBEAT_MSG_EVENT_COLD_MIGRATE_BEGIN) ) || + ( !event_type.compare(GUEST_HEARTBEAT_MSG_EVENT_REBOOT))) + { + if ( timer.tid ) + mtcTimer_stop ( timer ); + mtcTimer_start ( timer, guestTimer_handler, WAIT_FOR_INIT_TIMEOUT ); + ilog ("scheduling waiting_init transition in %d seconds\n", WAIT_FOR_INIT_TIMEOUT ); + } +} + +/* extend the reconnect time as the attempts pile up. max out at 1 minute. */ +void manage_reconnect_timeout ( instInfo * instInfo_ptr ) +{ + /* extend the reconnect time as the attempts pile up. max out at 1 minute. */ + if ( (instInfo_ptr->connect_wait_in_secs*2) > MTC_MINS_1 ) + instInfo_ptr->connect_wait_in_secs = MTC_MINS_1 ; + else + instInfo_ptr->connect_wait_in_secs *= 2 ; +} + +int connect_count = 0 ; +int guestInstClass::connect_handler ( struct guestInstClass::inst * inst_ptr ) +{ + int rc = PASS ; + switch ( inst_ptr->connectStage ) + { + case INST_CONNECT__START: + { + if ( inst_ptr->instance.connected == true ) + { + inst_ptr->connectStage = INST_CONNECT__START ; + inst_ptr->action = FSM_ACTION__NONE ; + + if (inst_ptr->connect_timer.tid) + mtcTimer_stop ( inst_ptr->connect_timer ); + } + else + { + ilog ("%s connect attempt in %d seconds\n", + log_prefix(&inst_ptr->instance).c_str(), inst_ptr->instance.connect_wait_in_secs); + inst_ptr->instance.connecting = true ; + mtcTimer_start ( inst_ptr->connect_timer, guestTimer_handler, inst_ptr->instance.connect_wait_in_secs ); + inst_ptr->connectStage = INST_CONNECT__WAIT ; + } + break ; + } + case INST_CONNECT__WAIT: + { + if ( inst_ptr->instance.connecting != true ) + { + slog ("%s bad connect wait state ; auto correcting\n", + log_prefix(&inst_ptr->instance).c_str()); + + inst_ptr->connectStage = INST_CONNECT__START ; + inst_ptr->action = FSM_ACTION__NONE ; + } + + else if ( inst_ptr->connect_timer.ring == true ) + { + char buf[PATH_MAX]; + + inst_ptr->connect_timer.ring = false ; + + /* if the socket is not there then don't try and connect to it */ + snprintf(buf, sizeof(buf), "%s/cgcs.heartbeat.%s.sock", QEMU_CHANNEL_DIR, inst_ptr->instance.uuid.data()); + if ( daemon_is_file_present ( buf ) ) + { + /* Try to connect with virtio_channel_connect ... + * If that succeeds then go DONE. + * if that fails with a ENOENT hen that means the socket fd is gone do close and delete instance + * otherwise retry the connect + */ + + ilog ( "%s connect start\n", log_prefix(&inst_ptr->instance).c_str()); + rc = virtio_channel_connect ( &inst_ptr->instance ); + if ( rc == PASS ) + { + inst_ptr->connectStage = INST_CONNECT__DONE ; + break ; + } + /* Abort connect if the instance channel is no longer there. + * -1 and errno=2 : No such file or directory) */ + else if (( rc == -1 ) && ( errno == ENOENT )) + { + ilog ("%s channel gone\n", log_prefix(&inst_ptr->instance).c_str() ); + del_inst ( inst_ptr->instance.uuid ); + return (RETRY); + } + else + { + wlog ("%s channel connect failed\n", + log_prefix(&inst_ptr->instance).c_str() ); + manage_reconnect_timeout ( &inst_ptr->instance ); + } + } + else + { + ilog ("%s does not exist\n", buf ); + manage_reconnect_timeout ( &inst_ptr->instance ); + } + inst_ptr->connectStage = INST_CONNECT__START ; + } + break ; + } + case INST_CONNECT__DONE: + { + + inst_ptr->connectStage = INST_CONNECT__START ; + inst_ptr->action = FSM_ACTION__NONE ; + + inst_ptr->instance.connecting = false ; + inst_ptr->instance.connected = true ; + + failure_reporting_count = 0 ; + + /* no longer failed */ + inst_ptr->instance.heartbeat.failed = false ; + inst_ptr->instance.heartbeat.b2b_misses = 0 ; + + /* waiting for init message */ + beatStateChange ( &inst_ptr->instance, hbs_server_waiting_init ) ; + + /* default back to the start 2 second reconnect time default */ + inst_ptr->instance.connect_wait_in_secs = DEFAULT_CONNECT_WAIT ; + + start_monitor_timer ( inst_ptr ); + + if ( inst_ptr->reconnect_timer.tid ) + mtcTimer_stop ( inst_ptr->reconnect_timer ); + mtcTimer_start ( inst_ptr->reconnect_timer, guestTimer_handler, HEARTBEAT_START_TIMEOUT ); + + ilog ("%s connect done\n", log_prefix(&inst_ptr->instance).c_str()); + + break ; + } + default: + { + slog ("Unsupported connect stage (%d) ... correcting\n", inst_ptr->connectStage ); + inst_ptr->connectStage = INST_CONNECT__START ; + } + } + return(rc); +} + +int guestInstClass::monitor_handler ( struct guestInstClass::inst * inst_ptr ) +{ + int rc = PASS ; + +#ifdef WANT_THIS + clog ("%s in '%s:%s' state - stage %d - R:%c F:%c H:%c\n", + log_prefix(&inst_ptr->instance).c_str(), + hb_get_state_name(inst_ptr->instance.hbState), + hb_get_state_name(inst_ptr->instance.vnState), + inst_ptr->monitorStage, + inst_ptr->instance.heartbeat.reporting ? 'Y' : 'n', + inst_ptr->instance.heartbeat.failed ? 'Y' : 'n', + inst_ptr->instance.heartbeating ? 'Y' : 'n'); + // inst_ptr->instance.heartbeat.waiting ? 'Y' : 'n'); +#endif + + switch ( inst_ptr->monitorStage ) + { + case INST_MONITOR__STEADY: + { + /* Manage Reconnect Timer */ + if ( inst_ptr->reconnect_timer.ring == true ) + { + inst_ptr->reconnect_timer.ring = false ; + if (( inst_ptr->instance.heartbeating == false ) && + ( inst_ptr->instance.connecting == false )) + { + /* If this timer rings and heartbeating is not started + * then we need to close the connection and repoen it + * Since the re-open is automatic all we need to do is + * close it here */ + wlog ("%s issuing auto-reconnect ; no heartbeating\n", + log_prefix(&inst_ptr->instance).c_str() ); + + reconnect_start ( inst_ptr->instance.uuid.data() ); + } + mtcTimer_start ( inst_ptr->reconnect_timer, guestTimer_handler, HEARTBEAT_START_TIMEOUT ); + } + + /* Manage Monitor Timer - expires in 3 cases + * 1. heartbeat miss - hbs_server_waiting_response + * 2. heartbeat done - hbs_server_waiting_challenge - interval is done and ready for the next one + * 3. heartbeat none - not heartbeating ; waiting for init + * 4. heratbeat fail - in wrong state + **/ + if ( inst_ptr->monitor_timer.ring == true ) + { + inst_ptr->monitor_timer.ring = false ; + + /* Case 1: heartbeat miss while waiting for heartbeat response */ + if ( inst_ptr->instance.hbState == hbs_server_waiting_response ) + { + int threshold = daemon_get_cfg_ptr()->hbs_failure_threshold ; + if (( inst_ptr->instance.heartbeat.failed == true ) || + ( inst_ptr->instance.heartbeat.reporting == false )) + { + hbStatusChange ( &inst_ptr->instance, false ); + beatStateChange ( &inst_ptr->instance, hbs_server_waiting_init) ; + } + else if ( ++inst_ptr->instance.heartbeat.b2b_misses > threshold ) + { + inst_ptr->instance.message_count = 0 ; + inst_ptr->instance.heartbeat.b2b_misses = 0 ; + + elog ("%s *** Heartbeat Loss *** (Timeout=%d msec)\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.heartbeat_interval_ms ); + + manage_heartbeat_failure ( &inst_ptr->instance ); + inst_ptr->monitorStage = INST_MONITOR__FAILURE ; + } + else + { + wlog ("%s *** Heartbeat Miss *** %d of %d (Timeout=%d msec)\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.heartbeat.b2b_misses, + threshold, + inst_ptr->instance.heartbeat_interval_ms ); + /* Send another challenge */ + send_challenge ( inst_ptr ) ; + } + } + + /* Case 2: Heartbeat done and the interval is expired. + * Just start another challenge request + */ + else if (( inst_ptr->instance.hbState != hbs_server_waiting_init ) && + ( inst_ptr->instance.hbState != hbs_server_waiting_response) && + ( inst_ptr->instance.heartbeat.waiting == false )) + { + // printf ("*"); + /* Send another challenge */ + inst_ptr->instance.heartbeat.b2b_misses = 0 ; + send_challenge ( inst_ptr ) ; + } + + /* Case 3: The monitor timer still runs while we are in the + * waiting for init state so just make sure we are + * handling init stuff + */ + else if ( inst_ptr->instance.hbState == hbs_server_waiting_init ) + { + clog ("%s is %s\n", log_prefix(&inst_ptr->instance).c_str(), + hb_get_state_name(inst_ptr->instance.hbState)); + inst_ptr->messageStage = INST_MESSAGE__RECEIVE ; + inst_ptr->instance.message_count = 0 ; + inst_ptr->instance.heartbeat.b2b_misses = 0 ; + } + + /* Case 4: Heratbeat has failed while we are in the wrong state */ + else + { + int threshold = daemon_get_cfg_ptr()->hbs_failure_threshold ; + if ( inst_ptr->instance.heartbeat.failed == true ) + { + ; /* nothing to do while failed */ + } + else if ( inst_ptr->instance.heartbeat.reporting == false ) + { + /* Send a challenge to keep the heartbeat going */ + send_challenge ( inst_ptr ) ; + } + else if ( ++inst_ptr->instance.heartbeat.b2b_misses > threshold ) + { + inst_ptr->instance.message_count = 0 ; + inst_ptr->instance.heartbeat.b2b_misses = 0 ; + + elog ("%s *** Heartbeat Loss *** (state:%s)\n", + log_prefix(&inst_ptr->instance).c_str(), + hb_get_state_name(inst_ptr->instance.hbState)); + + manage_heartbeat_failure ( &inst_ptr->instance ); + inst_ptr->monitorStage = INST_MONITOR__FAILURE ; + } + else + { + wlog ("%s *** Heartbeat Miss *** (state:%s)\n", + log_prefix(&inst_ptr->instance).c_str(), + hb_get_state_name(inst_ptr->instance.hbState)); + /* Send another challenge */ + send_challenge ( inst_ptr ) ; + } + } + } + + if ( inst_ptr->vote_timer.ring == true ) + { + if ( inst_ptr->instance.vnState == hbs_client_waiting_shutdown_response ) + { + // handle time out as silent agreement to accept + if ( !inst_ptr->instance.msg_type.compare(GUEST_HEARTBEAT_MSG_ACTION_NOTIFY) || + !inst_ptr->instance.msg_type.compare(GUEST_HEARTBEAT_MSG_ACTION_RESPONSE) ) + { + ilog ("%s response time out on '%s' message ; proceeding with action\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.msg_type.c_str()); + + string reject_reason = ""; + string vote_result = GUEST_HEARTBEAT_MSG_VOTE_RESULT_UNKNOWN; + if (!inst_ptr->instance.notification_type.compare(GUEST_HEARTBEAT_MSG_NOTIFY_REVOCABLE)) + { + vote_result = GUEST_HEARTBEAT_MSG_VOTE_RESULT_ACCEPT; + } + else if (!inst_ptr->instance.notification_type.compare(GUEST_HEARTBEAT_MSG_NOTIFY_IRREVOCABLE)) + { + vote_result = GUEST_HEARTBEAT_MSG_VOTE_RESULT_COMPLETE; + } + else + { + wlog ("%s Unexpected '%s' notify timeout ; proceeding with action\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.notification_type.c_str()); + } + send_vote_notify_resp (get_ctrl_ptr()->hostname, + inst_ptr->instance.uuid, + inst_ptr->instance.notification_type, + inst_ptr->instance.event_type, + vote_result, reject_reason); + } + + _schedule_init_timer ( inst_ptr->instance.event_type , + inst_ptr->init_timer) ; + + voteStateChange ( &inst_ptr->instance, hbs_server_waiting_init ); + } + inst_ptr->vote_timer.ring = false ; + } + break ; + } + case INST_MONITOR__DELAY: + { + if ( inst_ptr->monitor_timer.ring == true ) + { + inst_ptr->monitorStage = INST_MONITOR__FAILURE ; + } + break ; + } + case INST_MONITOR__FAILURE: + { + if ( get_instInv_ptr()->reporting == false ) + { + wlog_throttled (failure_reporting_count, 100, "host level reporting is disabled\n"); + } + else if ( inst_ptr->instance.heartbeat.reporting == false ) + { + wlog_throttled (failure_reporting_count, 100, "%s instance level reporting is disabled\n", + log_prefix(&inst_ptr->instance).c_str()); + } + else + { + inst_ptr->instance.heartbeat.failures++ ; + + wlog ("%s sending failure notification to guestAgent (failures:%d)\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.heartbeat.failures); + + string payload = "" ; + payload.append ("{\"hostname\":\""); + payload.append (get_ctrl_ptr()->hostname); + payload.append ("\",\"uuid\":\""); + payload.append (inst_ptr->instance.uuid); + payload.append ("\"}"); + + jlog1 ("%s Failure Event Payload: %s\n", + log_prefix(&inst_ptr->instance).c_str(), payload.c_str()); + + send_to_guestAgent ( MTC_EVENT_HEARTBEAT_LOSS , payload.data()); + failure_reporting_count = 0 ; + } + // inst_ptr->instance.heartbeat.failed = false ; + inst_ptr->monitorStage = INST_MONITOR__STEADY ; + + break ; + } + default: + { + inst_ptr->monitorStage = INST_MONITOR__STEADY ; + break ; + } + } + + /* This will try to reconnect failed channels */ + if (( !inst_ptr->instance.connected ) || + (( inst_ptr->instance.chan_fd > 0 ) && ( inst_ptr->instance.chan_ok != true ))) + { + if ( inst_ptr->action == FSM_ACTION__NONE ) + { + ilog ("%s enabling connect FSM\n", log_prefix(&inst_ptr->instance).c_str()); + + hbStatusChange ( &inst_ptr->instance, false) ; + + inst_ptr->connectStage = INST_CONNECT__START ; + inst_ptr->action = FSM_ACTION__CONNECT ; + } + else if ( inst_ptr->action != FSM_ACTION__CONNECT ) + { + wlog ("%s bypassing reconnect due to existing action (%d)\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->action); + } + } + + return (rc); +} + + +/***************************************************************************** + * + * Name : message_handler + * + * Purpose : Receive messages from the guest and trigger actions + * based on message content and type. + * + * Description: Only stage presently supported is INST_MESSAGE__RECEIVE + * for each connected socket. This FSM handler is not called + * unless there is a valid receive message to be handled. If + * for some reason there are no enqued messages then the FSM + * just returns having done thinting ; should not happen + * through. + * + * Currently supported message types are. + * + * GUEST_HEARTBEAT_MSG_INIT - vm heartbeat init message. + * > Action is to send an init_ack message to start heartbeating + * + * GUEST_HEARTBEAT_MSG_CHALLENGE_RESPONSE - a challenge response message + * > Action is to change state to 'hbs_server_waiting_challenge' + * and allow the heartbeat interval timer to expire in the + * monitor_handler which will then send another challenge + * request setting state back to 'hbs_server_waiting_response' + * + * Note: Unsupported messages are popped off the queue and discarded with + * an error log containing the message type. + * + *****************************************************************************/ +int guestInstClass::message_handler ( struct guestInstClass::inst * inst_ptr ) +{ + int rc = PASS ; + + switch ( inst_ptr->messageStage ) + { + case INST_MESSAGE__RECEIVE: + { + /* Only process if there are messages */ + if ( inst_ptr->message_list.size() ) + { + struct json_object *jobj_msg = inst_ptr->message_list.front(); + inst_ptr->message_list.pop_front(); + + if (jsonUtil_get_int(jobj_msg, GUEST_HEARTBEAT_MSG_VERSION, &inst_ptr->instance.version) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_VERSION, jobj_msg); + return FAIL; + } + if (jsonUtil_get_int(jobj_msg, GUEST_HEARTBEAT_MSG_REVISION, &inst_ptr->instance.revision) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_REVISION, jobj_msg); + return FAIL; + } + if (jsonUtil_get_string(jobj_msg, GUEST_HEARTBEAT_MSG_MSG_TYPE, &inst_ptr->instance.msg_type) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_MSG_TYPE, jobj_msg); + return FAIL; + } + if (jsonUtil_get_int(jobj_msg, GUEST_HEARTBEAT_MSG_SEQUENCE, &inst_ptr->instance.sequence) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_SEQUENCE, jobj_msg); + return FAIL; + } + + mlog1 ("%s:%s message - Seq:%x Ver:%d.%d Fd:%d\n", + inst_ptr->instance.uuid.c_str(), + inst_ptr->instance.msg_type.c_str(), + inst_ptr->instance.sequence , + inst_ptr->instance.version, inst_ptr->instance.revision, + inst_ptr->instance.chan_fd); + + if ( !inst_ptr->instance.msg_type.compare(GUEST_HEARTBEAT_MSG_CHALLENGE_RESPONSE) ) + { + if ( inst_ptr->instance.hbState == hbs_server_waiting_response ) + { + uint32_t heartbeat_response; + string heartbeat_health; + string corrective_action; + string log_msg; + + inst_ptr->instance.heartbeat.waiting = false ; + + if ( daemon_get_cfg_ptr()->debug_work ) + printf ("-"); + + inst_ptr->heartbeat_count++ ; + if (jsonUtil_get_int(jobj_msg, GUEST_HEARTBEAT_MSG_HEARTBEAT_RESPONSE, &heartbeat_response) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_HEARTBEAT_RESPONSE, jobj_msg); + return FAIL; + } + if (jsonUtil_get_string(jobj_msg, GUEST_HEARTBEAT_MSG_HEARTBEAT_HEALTH, &heartbeat_health) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_HEARTBEAT_HEALTH, jobj_msg); + return FAIL; + } + if (jsonUtil_get_string(jobj_msg, GUEST_HEARTBEAT_MSG_CORRECTIVE_ACTION, &corrective_action) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_CORRECTIVE_ACTION, jobj_msg); + return FAIL; + } + if (jsonUtil_get_string(jobj_msg, GUEST_HEARTBEAT_MSG_LOG_MSG, &log_msg) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_LOG_MSG, jobj_msg); + return FAIL; + } + + if ( heartbeat_response != inst_ptr->instance.heartbeat_challenge) + { + inst_ptr->instance.health_count = 0 ; + wlog_throttled (inst_ptr->mismatch_count, 100, "%s challenge secret mismatch (%d:%d) (throttle:100)\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.heartbeat_challenge, + heartbeat_response); + } + else if (!heartbeat_health.compare(GUEST_HEARTBEAT_MSG_HEALTHY)) + { + inst_ptr->mismatch_count = 0 ; + inst_ptr->instance.health_count = 0 ; + inst_ptr->instance.corrective_action_count = 0 ; + + mlog ("%s recv '%s' (seq:%x) (health:%s)\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.msg_type.c_str(), inst_ptr->instance.sequence, heartbeat_health.c_str()); + + /* lets wait for the period timer to expire before + * sending another in the monitor_handler */ + beatStateChange ( &inst_ptr->instance, hbs_server_waiting_challenge ) ; + + if ( inst_ptr->instance.heartbeating != true ) + { + hbStatusChange ( &inst_ptr->instance, true ); + } + + if (inst_ptr->instance.heartbeat.failed != false ) + { + inst_ptr->instance.heartbeat.failed = false ; + } + + ilog_throttled ( inst_ptr->instance.message_count, 1000, "%s is heartbeating ...(seq:%08x)\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.sequence ); + } + else + { + const char *msg = json_object_to_json_string_ext(jobj_msg, JSON_C_TO_STRING_PLAIN); + ilog ("%s received unhealthy response message: %s\n", + log_prefix(&inst_ptr->instance).c_str(), msg ); + + inst_ptr->mismatch_count = 0 ; + + /* lets wait for the period timer to expire before + * sending another in the monitor_handler */ + beatStateChange ( &inst_ptr->instance, hbs_server_waiting_challenge ) ; + + if ( inst_ptr->instance.health_count == 0 ) + { + if ( heartbeat_health.compare(GUEST_HEARTBEAT_MSG_UNHEALTHY) != 0 ) + { + wlog ("%s Invalid health reported (%s)\n", + log_prefix(&inst_ptr->instance).c_str(), + heartbeat_health.c_str() ); + } + + wlog_throttled ( inst_ptr->instance.health_count, 500, + "%s VM Unhealthy Message:\n", + log_prefix(&inst_ptr->instance).c_str()); + + wlog ("%s ... %s\n", log_prefix(&inst_ptr->instance).c_str(), + log_msg.c_str() ); + } + + inst_ptr->instance.unhealthy_corrective_action = corrective_action; + + if (!inst_ptr->instance.unhealthy_corrective_action.compare(GUEST_HEARTBEAT_MSG_ACTION_NONE) || + !inst_ptr->instance.unhealthy_corrective_action.compare(GUEST_HEARTBEAT_MSG_ACTION_UNKNOWN)) + { + wlog_throttled ( inst_ptr->instance.corrective_action_count, 500, + "%s corrective action is %s ; not reporting\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.unhealthy_corrective_action.c_str()); + } else { + inst_ptr->instance.unhealthy_failure = true ; + string payload = guestUtil_set_inst_info ( get_ctrl_ptr()->hostname , &inst_ptr->instance ); + inst_ptr->instance.unhealthy_failure = false ; + + ilog ("%s ill health notification\n", log_prefix(&inst_ptr->instance).c_str()); + send_to_guestAgent ( MTC_EVENT_HEARTBEAT_ILLHEALTH, payload.data()); + inst_ptr->instance.corrective_action_count = 0 ; + } + } + } + else if ( inst_ptr->instance.hbState == hbs_server_waiting_challenge ) + { + wlog ("%s received late '%s' response (seq:%x)\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.msg_type.c_str(), + inst_ptr->instance.sequence); + } + else + { + dlog ("%s recv '%s' while in '%s' state (seq:%x)\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.msg_type.c_str(), + hb_get_state_name(inst_ptr->instance.hbState), + inst_ptr->instance.sequence); + } + } + + else if ( !inst_ptr->instance.msg_type.compare(GUEST_HEARTBEAT_MSG_INIT) ) + { + const char *msg = json_object_to_json_string_ext(jobj_msg, JSON_C_TO_STRING_PLAIN); + ilog ("%s received init message: %s\n", + log_prefix(&inst_ptr->instance).c_str(), msg ); + + if (inst_ptr->instance.hbState != hbs_server_waiting_init) + { + wlog("%s unexpected 'init' message ; currState: '%s' (%d)\n", + log_prefix(&inst_ptr->instance).c_str(), + hb_get_state_name(inst_ptr->instance.hbState), + inst_ptr->instance.hbState ); + + /* Allow the heartbeat challenge response message log */ + inst_ptr->instance.message_count = 0 ; + beatStateChange ( &inst_ptr->instance, hbs_server_waiting_init ) ; + } + else + { + string instance_name; + string response; + + if (jsonUtil_get_int(jobj_msg, GUEST_HEARTBEAT_MSG_INVOCATION_ID, &inst_ptr->instance.invocation_id) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_INVOCATION_ID, jobj_msg); + return FAIL; + } + + if (jsonUtil_get_string(jobj_msg, GUEST_HEARTBEAT_MSG_NAME, &instance_name) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_NAME, jobj_msg); + return FAIL; + } + + if (jsonUtil_get_string(jobj_msg, GUEST_HEARTBEAT_MSG_CORRECTIVE_ACTION, &inst_ptr->instance.corrective_action) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_CORRECTIVE_ACTION, jobj_msg); + return FAIL; + } + + if (jsonUtil_get_int(jobj_msg, GUEST_HEARTBEAT_MSG_HEARTBEAT_INTERVAL_MS, &inst_ptr->instance.heartbeat_interval_ms) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_HEARTBEAT_INTERVAL_MS, jobj_msg); + return FAIL; + } + + if (jsonUtil_get_int(jobj_msg, GUEST_HEARTBEAT_MSG_VOTE_SECS, &inst_ptr->instance.vote_secs) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_VOTE_SECS, jobj_msg); + return FAIL; + } + + if (jsonUtil_get_int(jobj_msg, GUEST_HEARTBEAT_MSG_SHUTDOWN_NOTICE_SECS, &inst_ptr->instance.shutdown_notice_secs) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_SHUTDOWN_NOTICE_SECS, jobj_msg); + return FAIL; + } + + if (jsonUtil_get_int(jobj_msg, GUEST_HEARTBEAT_MSG_SUSPEND_NOTICE_SECS, &inst_ptr->instance.suspend_notice_secs) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_SUSPEND_NOTICE_SECS, jobj_msg); + return FAIL; + } + + if (jsonUtil_get_int(jobj_msg, GUEST_HEARTBEAT_MSG_RESUME_NOTICE_SECS, &inst_ptr->instance.resume_notice_secs) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_RESUME_NOTICE_SECS, jobj_msg); + return FAIL; + } + + if (jsonUtil_get_int(jobj_msg, GUEST_HEARTBEAT_MSG_RESTART_SECS, &inst_ptr->instance.restart_secs) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_RESTART_SECS, jobj_msg); + return FAIL; + } + + inst_ptr->instance.name = instance_name; + + /* Override the unused 'inst' name with an abbreviated version of the instance uuid + * cgcs.heartbeat.1f0bc3e3-efbe-48b8-9688-4821fc0ff83c.sock + * + * */ + if ( inst_ptr->instance.uuid.length() >= (24+12) ) + inst_ptr->instance.inst = inst_ptr->instance.uuid.substr(24,12); + + string name = log_prefix(&inst_ptr->instance).c_str() ; + + ilog ("%s 'init' message ; sending 'init_ack' (ver:%d.%d)\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.version, + inst_ptr->instance.revision ); + + inst_ptr->instance.heartbeat_challenge = rand(); + + + /* Set the unhealthy corrective action to unknown by default */ + inst_ptr->instance.unhealthy_corrective_action = GUEST_HEARTBEAT_MSG_ACTION_UNKNOWN ; + + ilog ("%s corrective_action = %s\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.corrective_action.c_str() ); + + ilog ("%s Interval : %4d msec\n",name.c_str(), inst_ptr->instance.heartbeat_interval_ms); + + /* auto correct an interval that is too small */ + if ( inst_ptr->instance.heartbeat_interval_ms < (uint32_t)daemon_get_cfg_ptr()->hbs_pulse_period ) + { + wlog ("%s cannot have an interval of zero seconds\n", + log_prefix(&inst_ptr->instance).c_str()); + + wlog ("%s ... auto correcting to %d msecs\n", + log_prefix(&inst_ptr->instance).c_str(), + daemon_get_cfg_ptr()->hbs_pulse_period); + + inst_ptr->instance.heartbeat_interval_ms = daemon_get_cfg_ptr()->hbs_pulse_period ; + } + + ilog ("%s Vote TO : %4d secs\n",name.c_str(), inst_ptr->instance.vote_secs); + inst_ptr->instance.vote_to_str = time_in_secs_to_str(inst_ptr->instance.vote_secs) ; + + ilog ("%s Shutdown : %4d secs\n", name.c_str(), inst_ptr->instance.shutdown_notice_secs); + inst_ptr->instance.shutdown_to_str = time_in_secs_to_str (inst_ptr->instance.shutdown_notice_secs); + + ilog ("%s Suspend : %4d secs\n", name.c_str(), inst_ptr->instance.suspend_notice_secs); + inst_ptr->instance.suspend_to_str = time_in_secs_to_str (inst_ptr->instance.suspend_notice_secs); + + ilog ("%s Resume : %4d secs\n", name.c_str(), inst_ptr->instance.resume_notice_secs); + inst_ptr->instance.resume_to_str = time_in_secs_to_str (inst_ptr->instance.resume_notice_secs); + + ilog ("%s Restart : %4d secs\n", name.c_str(), inst_ptr->instance.restart_secs); + inst_ptr->instance.restart_to_str = time_in_secs_to_str(inst_ptr->instance.restart_secs); + + /* cancel the init timer since we already got the init */ + if ( inst_ptr->init_timer.tid ) + mtcTimer_stop ( inst_ptr->init_timer ) ; + + /************************************************************* + * + * Send INIT ACK right away followed by the first Challenge. + * + * Cannot allow the FSM to run or we might see a + * race condition with another INIT messages that come after. + * + *************************************************************/ + response = guestSvrMsg_hdr_init(inst_ptr->instance.uuid , GUEST_HEARTBEAT_MSG_INIT_ACK); + response.append ("\""); + response.append (GUEST_HEARTBEAT_MSG_INVOCATION_ID); + response.append ("\":"); + response.append (int_to_string(inst_ptr->instance.invocation_id)); + response.append ("}\n"); + + inst_ptr->instance.message_count = 0 ; + + /* Send message to the vm through the libvirt channel */ + ilog("%s sending 'init_ack' invocation_id:%d, msg: %s\n", name.c_str(), + inst_ptr->instance.invocation_id, response.c_str()); + + get_instInv_ptr()->write_inst (&inst_ptr->instance, response.c_str(), response.length()); + + /* Send a challenge right away */ + beatStateChange ( &inst_ptr->instance, hbs_server_waiting_response ) ; + inst_ptr->instance.heartbeat.b2b_misses = 0 ; + inst_ptr->instance.heartbeat.failed = false ; + send_challenge ( inst_ptr ) ; + inst_ptr->messageStage = INST_MESSAGE__RECEIVE ; + } + } + else if ( !inst_ptr->instance.msg_type.compare(GUEST_HEARTBEAT_MSG_ACTION_RESPONSE) ) + { + uint32_t invocation_id; + const char *msg = json_object_to_json_string_ext(jobj_msg, JSON_C_TO_STRING_PLAIN); + ilog ("%s received action response message: %s\n", + log_prefix(&inst_ptr->instance).c_str(), msg ); + + if (jsonUtil_get_int(jobj_msg, GUEST_HEARTBEAT_MSG_INVOCATION_ID, &invocation_id) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_INVOCATION_ID, jobj_msg); + return FAIL; + } + + if ( invocation_id != inst_ptr->instance.invocation_id ) + { + wlog ("%s invocation id mismatch (%x:%x) - dropping response\n", + log_prefix(&inst_ptr->instance).c_str(), + invocation_id, + inst_ptr->instance.invocation_id ); + string log_err = "Invocation id mismatch. Received: "; + log_err.append(int_to_string(invocation_id)); + log_err.append(" expect: "); + log_err.append(int_to_string(inst_ptr->instance.invocation_id)); + send_client_msg_nack(&inst_ptr->instance, log_err); + } + else + { + string event_type; + string notification_type; + string vote_result; + string reject_reason; + + if(jsonUtil_get_string(jobj_msg, GUEST_HEARTBEAT_MSG_EVENT_TYPE, &event_type) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_EVENT_TYPE, jobj_msg); + return FAIL; + } + if (jsonUtil_get_string(jobj_msg, GUEST_HEARTBEAT_MSG_NOTIFICATION_TYPE, ¬ification_type) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_NOTIFICATION_TYPE, jobj_msg); + return FAIL; + } + if (jsonUtil_get_string(jobj_msg, GUEST_HEARTBEAT_MSG_VOTE_RESULT, &vote_result) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_VOTE_RESULT, jobj_msg); + return FAIL; + } + if (jsonUtil_get_string(jobj_msg, GUEST_HEARTBEAT_MSG_LOG_MSG, &reject_reason) != PASS) + { + handle_parse_failure(inst_ptr, GUEST_HEARTBEAT_MSG_LOG_MSG, jobj_msg); + return FAIL; + } + + send_vote_notify_resp (get_ctrl_ptr()->hostname, + inst_ptr->instance.uuid, + notification_type, + event_type, + vote_result, + reject_reason); + + inst_ptr->monitorStage = INST_MONITOR__STEADY ; + + _schedule_init_timer ( event_type , inst_ptr->init_timer ); + + // if pause-accept or pause-complete) + if (!event_type.compare(GUEST_HEARTBEAT_MSG_EVENT_PAUSE) && + (!vote_result.compare(GUEST_HEARTBEAT_MSG_VOTE_RESULT_ACCEPT) || + !vote_result.compare(GUEST_HEARTBEAT_MSG_VOTE_RESULT_COMPLETE)) ) + { + beatStateChange ( &inst_ptr->instance, hbs_server_waiting_init ) ; + } + voteStateChange ( &inst_ptr->instance, hbs_server_waiting_init ) ; + + // cancel the vote timer + if ( inst_ptr->vote_timer.tid ) + mtcTimer_stop ( inst_ptr->vote_timer ); + inst_ptr->vote_timer.ring = false ; + } + } + else if ( !inst_ptr->instance.msg_type.compare(GUEST_HEARTBEAT_MSG_EXIT) ) + { + const char *msg = json_object_to_json_string_ext(jobj_msg, JSON_C_TO_STRING_PLAIN); + ilog ("%s received client exit request: %s\n", + log_prefix(&inst_ptr->instance).c_str(), msg ); + + /* Prevent a heartbeat loss in the case of a graceful exit + * by moving into the waiting_init state */ + beatStateChange ( &inst_ptr->instance, hbs_server_waiting_init ) ; + + hbStatusChange ( &inst_ptr->instance, false ); + } + else + { + elog ("%s unsupported message type: %s.\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.msg_type.c_str()); + + string log_err = "unsupported message type: "; + log_err.append(inst_ptr->instance.msg_type); + send_client_msg_nack(&inst_ptr->instance, log_err); + } + json_object_put(jobj_msg); + } + break ; + } + + default: + { + elog ("Unsupported stage (%d)\n", inst_ptr->messageStage ); + } + } + return (rc) ; +} + +/************************************************************************************* + * + * Name : send_challenge + * + * Description: Transmit a heartbeat challenge to he specified VM + * and start the timoeut timer. + * + **************************************************************************************/ +int guestInstClass::send_challenge ( struct guestInstClass::inst * inst_ptr ) +{ + size_t bytes_sent ; + + string message = guestSvrMsg_hdr_init(inst_ptr->instance.uuid , GUEST_HEARTBEAT_MSG_CHALLENGE); + + beatStateChange ( &inst_ptr->instance, hbs_server_waiting_response ); + + inst_ptr->instance.heartbeat_challenge = rand(); + + message.append ("\""); + message.append (GUEST_HEARTBEAT_MSG_HEARTBEAT_CHALLENGE); + message.append ("\":"); + message.append (int_to_string(inst_ptr->instance.heartbeat_challenge)); + message.append ("}\n"); + + /* Send message to the vm through the libvirt channel */ + bytes_sent = write_inst (&inst_ptr->instance, message.c_str(), message.length()); + + /* The write_inst will report an error log. + * This one is only to report a partial message send. + */ + if (( bytes_sent > 0) && ( bytes_sent != message.length())) + { + wlog ("%s only sent %ld of %ld bytes\n", + log_prefix(&inst_ptr->instance).c_str(), + bytes_sent, message.length() ); + } + + /* Waiting on a response now */ + inst_ptr->instance.heartbeat.waiting = true ; + + start_monitor_timer ( inst_ptr ) ; + + if ( daemon_get_cfg_ptr()->debug_work ) + printf ("_"); + + return (PASS); +} + +/************************************************************************************* + * + * Name : send_vote_notify + * + * Description: Send a voting or notification message to GuestClient on VM + * and start the timeout timer. + * + **************************************************************************************/ +int guestInstClass::send_vote_notify ( string uuid ) +{ + struct guestInstClass::inst * inst_ptr = getInst(uuid); + size_t bytes_sent ; + uint32_t timeout_ms; + + string message = guestSvrMsg_hdr_init(inst_ptr->instance.uuid , GUEST_HEARTBEAT_MSG_ACTION_NOTIFY); + + voteStateChange ( &inst_ptr->instance, hbs_client_waiting_shutdown_response ); + + + if ( !inst_ptr->instance.notification_type.compare(GUEST_HEARTBEAT_MSG_NOTIFY_REVOCABLE) ) + { + timeout_ms = inst_ptr->instance.vote_secs * 1000; + } + else + { + timeout_ms = inst_ptr->instance.vote_secs ; + if (!inst_ptr->instance.event_type.compare(GUEST_HEARTBEAT_MSG_EVENT_STOP) || + !inst_ptr->instance.event_type.compare(GUEST_HEARTBEAT_MSG_EVENT_REBOOT)) + { + timeout_ms = inst_ptr->instance.shutdown_notice_secs * 1000 ; + + } else if (!inst_ptr->instance.event_type.compare(GUEST_HEARTBEAT_MSG_EVENT_SUSPEND) || + !inst_ptr->instance.event_type.compare(GUEST_HEARTBEAT_MSG_EVENT_PAUSE) || + !inst_ptr->instance.event_type.compare(GUEST_HEARTBEAT_MSG_EVENT_RESIZE_BEGIN) || + !inst_ptr->instance.event_type.compare(GUEST_HEARTBEAT_MSG_EVENT_LIVE_MIGRATE_BEGIN) || + !inst_ptr->instance.event_type.compare(GUEST_HEARTBEAT_MSG_EVENT_COLD_MIGRATE_BEGIN) ) { + timeout_ms = inst_ptr->instance.suspend_notice_secs * 1000 ; + + } else if (!inst_ptr->instance.event_type.compare(GUEST_HEARTBEAT_MSG_EVENT_UNPAUSE) || + !inst_ptr->instance.event_type.compare(GUEST_HEARTBEAT_MSG_EVENT_RESUME) || + !inst_ptr->instance.event_type.compare(GUEST_HEARTBEAT_MSG_EVENT_RESIZE_END) || + !inst_ptr->instance.event_type.compare(GUEST_HEARTBEAT_MSG_EVENT_LIVE_MIGRATE_END) || + !inst_ptr->instance.event_type.compare(GUEST_HEARTBEAT_MSG_EVENT_COLD_MIGRATE_END) ) { + timeout_ms = inst_ptr->instance.resume_notice_secs * 1000 ; + } else { + wlog ("%s unsupported event type (%s) defaulting to 'vote' timeout of %d secs\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.event_type.c_str(), + inst_ptr->instance.vote_secs); + } + } + + dlog ("%s event_type:%s notification_type:%s invocation_id:%d timeout_ms:%d\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.event_type.c_str(), + inst_ptr->instance.notification_type.c_str(), + inst_ptr->instance.invocation_id, + timeout_ms); + + message.append ("\""); + message.append (GUEST_HEARTBEAT_MSG_INVOCATION_ID); + message.append ("\":"); + message.append (int_to_string(inst_ptr->instance.invocation_id)); + message.append (",\""); + message.append (GUEST_HEARTBEAT_MSG_EVENT_TYPE); + message.append ("\":\""); + message.append (inst_ptr->instance.event_type); + message.append ("\",\""); + message.append (GUEST_HEARTBEAT_MSG_NOTIFICATION_TYPE); + message.append ("\":\""); + message.append (inst_ptr->instance.notification_type); + message.append ("\",\""); + message.append (GUEST_HEARTBEAT_MSG_TIMEOUT_MS); + message.append ("\":"); + message.append (int_to_string(timeout_ms)); + message.append ("}\n"); + + ilog("%s send_vote_notify message=%s\n", + log_prefix(&inst_ptr->instance).c_str(), message.c_str()); + + /* Send message to the vm through the libvirt channel */ + bytes_sent = write_inst (&inst_ptr->instance, message.c_str(), message.length()); + if ( bytes_sent != message.length() ) + { + wlog ("%s only sent %ld of %ld bytes\n", inst_ptr->instance.inst.c_str(), + bytes_sent, message.length() ); + } + + if ( inst_ptr->vote_timer.tid ) + mtcTimer_stop ( inst_ptr->vote_timer ); + mtcTimer_start ( inst_ptr->vote_timer, guestTimer_handler, inst_ptr->instance.vote_secs ); + + dlog("%s timer started for %d seconds\n", + log_prefix(&inst_ptr->instance).c_str(), + inst_ptr->instance.vote_secs); + + return (PASS); +} + +/************************************************************************************* + * + * Name : send_vote_notify_resp + * + * Description: Send response for voting or notification to GuestAgent + * + **************************************************************************************/ +int guestInstClass::send_vote_notify_resp ( char * hostname, string uuid, + string notification_type, + string event_type, + string vote_result, + string reject_reason) +{ + instInfo * instInfo_ptr = get_inst ( uuid ); + + if ( !instInfo_ptr ) + { + elog ("%s is unknown\n", uuid.c_str()); + return FAIL; + } + + if (!vote_result.compare(GUEST_HEARTBEAT_MSG_VOTE_RESULT_ACCEPT) || + !vote_result.compare(GUEST_HEARTBEAT_MSG_VOTE_RESULT_COMPLETE)) + { + // accept + ilog ("%s '%s' '%s' '%s'\n", + log_prefix(instInfo_ptr).c_str(), + notification_type.c_str(), + event_type.c_str(), + vote_result.c_str()); + + if (!vote_result.compare(GUEST_HEARTBEAT_MSG_VOTE_RESULT_COMPLETE) && + !event_type.compare(GUEST_HEARTBEAT_MSG_EVENT_SUSPEND)) + { + instInfo_ptr->connected = false ; + hbStatusChange ( instInfo_ptr , false ); + } + } else if (!vote_result.compare(GUEST_HEARTBEAT_MSG_VOTE_RESULT_REJECT)) { + ilog ("%s '%s' '%s' '%s' reason: %s\n", + log_prefix(instInfo_ptr).c_str(), + notification_type.c_str(), + event_type.c_str(), + vote_result.c_str(), + reject_reason.c_str()); + } else if (!vote_result.compare(GUEST_HEARTBEAT_MSG_VOTE_RESULT_TIMEOUT)) { + ilog ("%s '%s' '%s' '%s'\n", + log_prefix(instInfo_ptr).c_str(), + notification_type.c_str(), + event_type.c_str(), + vote_result.c_str()); + } else if (!vote_result.compare(GUEST_HEARTBEAT_MSG_VOTE_RESULT_ERROR)) { + elog ("%s vote to '%s' returned error: %s\n", + log_prefix(instInfo_ptr).c_str(), + event_type.c_str(), + vote_result.c_str()); + } else { + elog ("%s vote to '%s' unknown vote response %s\n", + log_prefix(instInfo_ptr).c_str(), + event_type.c_str(), + vote_result.c_str()); + } + + string payload = "" ; + payload.append ("{\"hostname\":\""); + payload.append (hostname); + payload.append ("\", \"uuid\": \""); + payload.append (uuid.c_str()); + payload.append ("\", \"notification_type\": \""); + payload.append (notification_type); + payload.append ("\", \"event-type\": \""); + payload.append (event_type); + payload.append ("\", \"vote\": \""); + payload.append (vote_result); + payload.append ("\", \"reason\": \""); + payload.append (reject_reason); + payload.append ("\"}"); + + jlog ("%s Notification Event Payload: %s\n", log_prefix(instInfo_ptr).c_str(), payload.c_str()); + + send_to_guestAgent ( MTC_EVENT_VOTE_NOTIFY , payload.data()); + + return (PASS); +} + +/************************************************************************************* + * + * Name : send_client_msg_nack + * + * Description: Send failure response to GuestClient when fail to process the client message + * + **************************************************************************************/ +void guestInstClass::send_client_msg_nack ( instInfo * instInfo_ptr, + string log_err) +{ + size_t bytes_sent ; + + string message = guestSvrMsg_hdr_init(instInfo_ptr->uuid , GUEST_HEARTBEAT_MSG_NACK); + + message.append ("\""); + message.append (GUEST_HEARTBEAT_MSG_INVOCATION_ID); + message.append ("\":"); + message.append (int_to_string(instInfo_ptr->invocation_id)); + message.append (",\""); + message.append (GUEST_HEARTBEAT_MSG_LOG_MSG); + message.append ("\":\""); + message.append (log_err.c_str()); + message.append ("\"}\n"); + + ilog("%s send_client_msg_nack message=%s\n", + log_prefix(instInfo_ptr).c_str(), message.c_str()); + + /* Send message to the vm through the libvirt channel */ + bytes_sent = write_inst (instInfo_ptr, message.c_str(), message.length()); + if ( bytes_sent != message.length() ) + { + wlog ("%s only sent %ld of %ld bytes\n", instInfo_ptr->inst.c_str(), + bytes_sent, message.length() ); + } +} + +/************************************************************************************* + * + * Name : handle_parse_failure + * + * Description: Handle JSON parse failure + * + **************************************************************************************/ +void guestInstClass::handle_parse_failure ( struct guestInstClass::inst * inst_ptr, + const char *key, + struct json_object *jobj_msg) +{ + string log_err = "failed to parse "; + log_err.append(key); + elog("%s %s\n", log_prefix(&inst_ptr->instance).c_str(), log_err.c_str()); + send_client_msg_nack(&inst_ptr->instance, log_err); + /* pop_front() only deletes the internal copy of jobj_msg in the message_list. + The original object still needs to be released here */ + json_object_put(jobj_msg); +} diff --git a/mtce-guest/src/guestSvrMsg.cpp b/mtce-guest/src/guestSvrMsg.cpp new file mode 100644 index 00000000..c5b4fde3 --- /dev/null +++ b/mtce-guest/src/guestSvrMsg.cpp @@ -0,0 +1,784 @@ +/* + * Copyright (c) 2013-2018 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + + /** + * @file + * Wind River CGTS Platform Guest Heartbeat Server Daemon on Compute + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* for hostent */ +#include +#include +#include +#include +#include +#include +#include /* for close and usleep */ +#include /* for realtime scheduling api */ +#include + +using namespace std; + +#include "nodeBase.h" +#include "daemon_ini.h" /* Ini Parser Header */ +#include "daemon_common.h" /* Common definitions and types for daemons */ +#include "daemon_option.h" /* Common options for daemons */ +#include "nodeUtil.h" /* for ... common utilities */ +#include "jsonUtil.h" /* for ... jason utilities */ +#include "nodeTimers.h" /* for ... maintenance timers */ +#include "nodeMacro.h" /* for ... CREATE_NONBLOCK_INET_UDP_RX_SOCKET */ +#include "nodeEvent.h" /* for ... set_inotify_watch, set_inotify_close */ +#include "guestBase.h" +#include "guestInstClass.h" /* for ... guestUtil_inst_init */ +#include "guestUtil.h" /* for ... guestUtil_inst_init */ +#include "guestSvrUtil.h" /* for ... hb_get_message_type_name */ +#include "guestSvrMsg.h" /* for ... this module header */ + +extern void hbStatusChange ( instInfo * instInfo_ptr, bool status ); +extern void beatStateChange ( instInfo * instInfo_ptr , hb_state_t newState ); + +/***************************************************************************** + * + * Name : guestSvrMsg_hdr_init + * + * Purpose: Initialize the message header. Example output: + * {"version":2,"revision":1,"msg_type":"init","sequence":29, + * The rest of the message should be appended to it. + * + *****************************************************************************/ + +string guestSvrMsg_hdr_init (string channel, string msg_type) +{ + instInfo * instInfo_ptr = get_instInv_ptr()->get_inst (channel); + + string msg = "\n{\""; + msg.append(GUEST_HEARTBEAT_MSG_VERSION); + msg.append("\":"); + msg.append(int_to_string(GUEST_HEARTBEAT_MSG_VERSION_CURRENT)); + msg.append(",\""); + msg.append(GUEST_HEARTBEAT_MSG_REVISION); + msg.append("\":"); + msg.append(int_to_string(GUEST_HEARTBEAT_MSG_REVISION_CURRENT)); + msg.append(",\""); + msg.append(GUEST_HEARTBEAT_MSG_MSG_TYPE); + msg.append("\":\""); + msg.append(msg_type); + msg.append("\",\""); + msg.append(GUEST_HEARTBEAT_MSG_SEQUENCE); + msg.append("\":"); + msg.append(int_to_string(++(instInfo_ptr->sequence))); + msg.append(","); + + // store msg_type in instance structure so that it is available to handle timeout + instInfo_ptr->msg_type = msg_type; + return msg; +} + +/** + * Manages the fault reporting state + * - returns current reporting state + * */ +bool manage_reporting_state ( instInfo * instInfo_ptr, string state) +{ + if (!state.compare("enabled")) + { + if ( instInfo_ptr->heartbeat.reporting == false ) + { + ilog ("%s heartbeat reporting '%s' by guestAgent\n", + log_prefix(instInfo_ptr).c_str(), + state.c_str()); + + instInfo_ptr->heartbeat.reporting = true ; + instInfo_ptr->message_count = 0 ; + } + } + else + { + if ( instInfo_ptr->heartbeat.reporting == true ) + { + ilog ("%s heartbeat reporting '%s' by guestAgent\n", + log_prefix(instInfo_ptr).c_str(), + state.c_str()); + + instInfo_ptr->heartbeat.reporting = false ; + instInfo_ptr->message_count = 0 ; + hbStatusChange ( instInfo_ptr, false) ; /* heartbeating is now false */ + beatStateChange ( instInfo_ptr, hbs_server_waiting_init ) ; + } + } + + return instInfo_ptr->heartbeat.reporting ; +} + +/***************************************************************************** + * + * Name : guestAgent_qry_handler + * + * Purpose: Loop over all the instances and return their uuid, hostname, + * reporting state, heartbneating status and timeout values. + * + * { "hostname":"compute-1", "instances": [{"uuid":"","heartbeat":"", status":"}, timeouts ...]} + * + *****************************************************************************/ +int guestInstClass::guestAgent_qry_handler ( void ) +{ + int rc = PASS ; + + /* check for empty list condition */ + if ( inst_head ) + { + struct inst * inst_ptr = static_cast(NULL) ; + for ( inst_ptr = inst_head ; ; inst_ptr = inst_ptr->next ) + { + string payload = guestUtil_set_inst_info ( get_ctrl_ptr()->hostname , &inst_ptr->instance ); + jlog ("%s Query Instance Response:%ld:%s\n", + log_prefix(&inst_ptr->instance).c_str(), + payload.size(), + payload.c_str() ); + + if (( rc=send_to_guestAgent ( MTC_CMD_QRY_INST, payload.data())) != PASS ) + { + wlog ("%s failed to send query instance response to guestAgent\n", + log_prefix(&inst_ptr->instance).c_str()); + } + + /* Deal with exit case */ + if (( inst_ptr->next == NULL ) || ( inst_ptr == inst_tail )) + { + break ; + } + } + } + return (rc); +} + +/***************************************************************************** + * + * Name : recv_from_guestAgent + * + * Purpose: Handle guestAgent commands + * + * MTC_EVENT_LOOPBACK + * MTC_CMD_QRY_INST + * MTC_CMD_DEL_INST + * MTC_CMD_MOD_INST + * MTC_CMD_ADD_INST + * MTC_CMD_MOD_HOST + * + * ***************************************************************************/ +int recv_from_guestAgent ( unsigned int cmd, char * buf_ptr ) +{ + int rc = PASS ; + + mlog1 ("Cmd:%x - %s\n", cmd, buf_ptr); + + if ( cmd == MTC_EVENT_LOOPBACK ) + { + /* TODO: Send message back */ + return (rc) ; + } + else if ( cmd == MTC_CMD_QRY_INST ) + { + if ( ( rc = get_instInv_ptr()->qry_inst ()) != PASS ) + { + elog ("failed to send hosts instance info\n"); + } + return (rc) ; + } + else if ( cmd == MTC_CMD_VOTE_INST + || cmd == MTC_CMD_NOTIFY_INST ) + { + string source; + string uuid; + string event; + + rc = FAIL_KEY_VALUE_PARSE ; /* default to parse error */ + + if (( rc = jsonUtil_get_key_val ( buf_ptr, "source", source )) != PASS) + { + elog ("failed to extract 'source' (cmd:%x %s)\n", cmd , buf_ptr ); + } + else if (( rc = jsonUtil_get_key_val ( buf_ptr, "uuid", uuid )) != PASS) + { + elog ("failed to extract 'uuid' (cmd:%x %s)\n", cmd , buf_ptr); + } + else if (( rc = jsonUtil_get_key_val ( buf_ptr, "event", event )) != PASS) + { + elog ("failed to extract 'event' key (cmd:%x %s)\n", cmd , buf_ptr); + } + else + { + + // send message to guest Client + instInfo * instInfo_ptr = get_instInv_ptr()->get_inst(uuid); + if ( instInfo_ptr ) + { + /* If this is a resume then we need to reconnect to the channel */ + if ( !event.compare(GUEST_HEARTBEAT_MSG_EVENT_RESUME) ) + { + /* issue a reconnect if we are not connected the hartbeating has not started */ + if (( instInfo_ptr->connected == false ) || + ( instInfo_ptr->heartbeating == false )) + { + // instInfo_ptr->connect_wait_in_secs = 10 ; + get_instInv_ptr()->reconnect_start ( instInfo_ptr->uuid.data() ); + } + } + + instInfo_ptr->event_type = event; + if (MTC_CMD_VOTE_INST == cmd) + { + // for voting + instInfo_ptr->notification_type = GUEST_HEARTBEAT_MSG_NOTIFY_REVOCABLE ; + + ilog ("%s sending revocable '%s' vote\n", + log_prefix(instInfo_ptr).c_str(), + event.c_str()); + } + else + { + // for notification + instInfo_ptr->notification_type = GUEST_HEARTBEAT_MSG_NOTIFY_IRREVOCABLE ; + + ilog ("%s sending irrevocable '%s' notify\n", + log_prefix(instInfo_ptr).c_str(), + event.c_str()); + } + get_instInv_ptr()->send_vote_notify(uuid) ; + rc = PASS ; + } + else + { + wlog ("%s is unknown\n", uuid.c_str()); + } + } + } + else + { + string source ; + string uuid ; + string service ; + string state ; + + rc = FAIL_KEY_VALUE_PARSE ; /* default to parse error */ + + if (( rc = jsonUtil_get_key_val ( buf_ptr, "source", source )) != PASS) + { + elog ("failed to extract 'source' (cmd:%x %s)\n", cmd , buf_ptr ); + } + else if (( rc = jsonUtil_get_key_val ( buf_ptr, "uuid", uuid )) != PASS) + { + elog ("failed to extract 'uuid' (cmd:%x %s)\n", cmd , buf_ptr); + } + else if (( rc = jsonUtil_get_key_val ( buf_ptr, "service", service )) != PASS) + { + elog ("failed to extract 'service' key (cmd:%x %s)\n", cmd , buf_ptr); + } + else if (( rc = jsonUtil_get_key_val ( buf_ptr, "state", state )) != PASS) + { + elog ("failed to extract 'state' (cmd:%x %s)\n", cmd , buf_ptr ); + } + else + { + rc = RETRY ; + switch ( cmd ) + { + case MTC_CMD_DEL_INST: + { + ilog ("%s delete\n", uuid.c_str()); + + if ( get_instInv_ptr()->del_inst( uuid ) == PASS ) + { + rc = PASS ; + } + else + { + dlog ("%s delete failed ; uuid lookup\n", uuid.c_str()); + rc = FAIL_NOT_FOUND ; + } + if (daemon_get_cfg_ptr()->debug_level ) + get_instInv_ptr()->print_instances (); + break ; + } + case MTC_CMD_ADD_INST: + case MTC_CMD_MOD_INST: + { + instInfo * instInfo_ptr = get_instInv_ptr()->get_inst ( uuid ); + if ( instInfo_ptr ) + { + manage_reporting_state ( instInfo_ptr, state ); + rc = PASS ; + } + + /* if true then the current channel was not found and we need to add it */ + if ( rc == RETRY ) + { + instInfo instance ; + guestUtil_inst_init (&instance); + + instance.uuid = uuid ; + ilog ("%s add with %s reporting %s\n", + uuid.c_str(), + service.c_str(), + state.c_str()); + + get_instInv_ptr()->add_inst ( uuid, instance ); + instInfo * instInfo_ptr = get_instInv_ptr()->get_inst ( uuid ); + manage_reporting_state ( instInfo_ptr, state ); + } + if (daemon_get_cfg_ptr()->debug_level ) + get_instInv_ptr()->print_instances(); + + break ; + } + case MTC_CMD_MOD_HOST: + { + guestInstClass * obj_ptr = get_instInv_ptr() ; + string reporting_state = "" ; + rc = jsonUtil_get_key_val ( buf_ptr, "heartbeat", reporting_state ) ; + if ( rc != PASS) + { + elog ("failed to extract heartbeat reporting state (rc=%d)\n", rc ); + wlog ("... disabling 'heartbeat' fault reporting due to error\n"); + obj_ptr->reporting = false ; + rc = FAIL_JSON_PARSE ; + } + else if ( !reporting_state.compare("enabled") ) + { + ilog ("Enabling host level 'heartbeat' fault reporting\n"); + obj_ptr->reporting = true ; + } + else + { + ilog ("Disabling host level 'heartbeat' fault reporting\n"); + obj_ptr->reporting = false ; + } + break ; + } + default: + { + elog ("unsupported command (%x)\n", cmd ); + } + } + } + } + return (rc); +} + +/**************************************************************************** + * + * Name : send_to_guestAgent + * + * Purpose : Send a command and buffer to the guestAgent + * + * Description: If the guestAgent IP is not known the message is dropped + * and a retry is returned. Otherwise the supplied message is + * sent to the guestAgent running on the controller. + * + * **************************************************************************/ +int send_to_guestAgent ( unsigned int cmd, const char * buf_ptr ) +{ + int bytes = 0; + + ctrl_type * ctrl_ptr = get_ctrl_ptr () ; + + int rc = PASS ; + mtc_message_type mtc_cmd ; + memset (&mtc_cmd,0,sizeof(mtc_message_type)); + + memcpy ( &mtc_cmd.buf[0], buf_ptr, strlen(buf_ptr)); + bytes = sizeof(mtc_message_type) ; + + if ( ctrl_ptr->address_peer.empty()) + { + mlog2 ("controller address unknown ; dropping message (%x:%s)", cmd , buf_ptr ); + return RETRY ; + } + + mlog1 ("Sending: %s:%d Cmd:%x:%s\n", ctrl_ptr->address_peer.c_str(), ctrl_ptr->sock.agent_rx_port, cmd, buf_ptr ); + + mtc_cmd.cmd = cmd ; + + /* rc = message size */ + rc = ctrl_ptr->sock.server_tx_sock->write((char *)&mtc_cmd, bytes,ctrl_ptr->address_peer.c_str()); + + if ( 0 > rc ) + { + elog("failed to send (%d:%m)\n", errno ); + rc = FAIL_SOCKET_SENDTO ; + } + else + { + mlog1 ("Transmit to %14s port %d\n", + ctrl_ptr->address_peer.c_str(), + ctrl_ptr->sock.server_tx_sock->get_dst_addr()->getPort()); + print_mtc_message ( &mtc_cmd ); + rc = PASS ; + } + + return (rc); +} + +/********************************************************************************* + * + * Name : write_inst (guestInstClass::public) + * + * Purpose: Send a message to the specified VM instance. + * + *********************************************************************************/ +ssize_t guestInstClass::write_inst ( instInfo * instInfo_ptr, + const char * message, + size_t size) +{ + string name = log_prefix(instInfo_ptr); + + errno = 0 ; + size_t len = write ( instInfo_ptr->chan_fd, message, size ); + if ( len != size ) + { + if ( errno ) + { + wlog_throttled ( instInfo_ptr->failure_count, 100, + "%s failed to send '%s' (seq:%x) (%d:%m)\n", name.c_str(), + instInfo_ptr->msg_type.c_str(), + instInfo_ptr->sequence, errno ); + + if ( errno == EPIPE ) + { + instInfo_ptr->connected = false ; + + instInfo_ptr->connect_wait_in_secs = DEFAULT_CONNECT_WAIT ; + get_instInv_ptr()->reconnect_start ( instInfo_ptr->uuid.data() ); + } + + len = 0 ; + } + else + { + wlog_throttled ( instInfo_ptr->failure_count, 100, + "%s send '%s' (seq:%x) (len:%ld)\n", name.c_str(), + instInfo_ptr->msg_type.c_str(), + instInfo_ptr->sequence, len); + } + } + else + { + instInfo_ptr->failure_count = 0 ; + mlog("%s send '%s' (seq:%x)\n", name.c_str(), + instInfo_ptr->msg_type.c_str(), + instInfo_ptr->sequence ); + } + return (len); +} + +/********************************************************************************* + * + * Name : process_msg (guestInstClass::private) + * + * Purpose : process delimited message + * + *********************************************************************************/ +void guestInstClass::process_msg(json_object *jobj_msg, + struct guestInstClass::inst * inst_ptr) +{ + int version; + string msg_type; + string log_err = "failed to parse "; + guestInstClass * obj_ptr = get_instInv_ptr(); + + //parse incoming msg + if (jobj_msg == NULL) + { + wlog("%s\n", log_err.c_str()); + return; + } + + if (jsonUtil_get_int(jobj_msg, GUEST_HEARTBEAT_MSG_VERSION, &version) != PASS) + { + // fail to parse the version + log_err.append(GUEST_HEARTBEAT_MSG_VERSION); + elog("%s\n", log_err.c_str()); + obj_ptr->send_client_msg_nack(&inst_ptr->instance, log_err); + json_object_put(jobj_msg); + return; + } + + if ( version < GUEST_HEARTBEAT_MSG_VERSION_CURRENT) + { + char log_err_str[100]; + sprintf(log_err_str, "Bad version: %d, expect version: %d", + version, GUEST_HEARTBEAT_MSG_VERSION_CURRENT); + elog("%s\n", log_err_str); + log_err = log_err_str; + obj_ptr->send_client_msg_nack(&inst_ptr->instance, log_err); + json_object_put(jobj_msg); + return; + } + + if (jsonUtil_get_string(jobj_msg, GUEST_HEARTBEAT_MSG_MSG_TYPE, &msg_type) != PASS) + { + // fail to parse the msg_type + log_err.append(GUEST_HEARTBEAT_MSG_MSG_TYPE); + elog("%s\n", log_err.c_str()); + obj_ptr->send_client_msg_nack(&inst_ptr->instance, log_err); + json_object_put(jobj_msg); + return; + } + + /* Enqueue the message to its instance message list */ + inst_ptr->message_list.push_back(jobj_msg); +} + +/********************************************************************************* + * + * Name : parser (guestInstClass::private) + * + * Purpose : parse message segments and feed valid message to process_msg + * + *********************************************************************************/ +void guestInstClass::parser(char *buf, + ssize_t len, + json_tokener* tok, + int newline_found, + struct guestInstClass::inst * inst_ptr) +{ + json_object *jobj = json_tokener_parse_ex(tok, buf, len); + enum json_tokener_error jerr = json_tokener_get_error(tok); + + if (jerr == json_tokener_success) { + process_msg(jobj, inst_ptr); + return; + } + + else if (jerr == json_tokener_continue) { + // partial JSON is parsed , continue to read from socket. + if (newline_found) { + // if newline was found in the middle of the buffer, the message + // should be completed at this point. Throw out incomplete message + // by resetting tokener. + json_tokener_reset(tok); + } + } + else + { + // parsing error + json_tokener_reset(tok); + } +} + +/********************************************************************************* + * + * Name : handle_virtio_serial_msg (guestInstClass::private) + * + * Purpose : handle delimitation and assembly of message stream + * + * Description: Multiple messages from the host can be bundled together into a + * single "read" so we need to check message boundaries and handle + * breaking the message apart. Assume a valid message does not + * contain newline '\n', and newline is added to the beginning and + * end of each message by the sender to delimit the boundaries. + * + *********************************************************************************/ +void guestInstClass::handle_virtio_serial_msg( + char *buf, + ssize_t len, + json_tokener* tok, + struct guestInstClass::inst * inst_ptr) +{ + char *newline; + ssize_t len_head; + +next: + if (len <= 0) + return; + + // search for newline as delimiter + newline = (char *)memchr((char *)buf, '\n', len); + + if (newline) { + // split buffer to head and tail at the location of newline. + // feed the head to the parser and recursively process the tail. + len_head = newline-buf; + + // parse head + if (len_head > 0) + parser(buf, len_head, tok, 1, inst_ptr); + + // start of the tail: skip newline + buf += len_head+1; + // length of the tail: deduct 1 for the newline character + len -= len_head+1; + + // continue to process the tail. + goto next; + } + else { + parser(buf, len, tok, 0, inst_ptr); + } +} + +/********************************************************************************* + * + * Name : readInst (guestInstClass::private) + * + * Purpose : try to receive a single message from all instances. + * + * Description: Each received message is enqueued into the associated + * instance's message queue. + * + *********************************************************************************/ + +int fail_count = 0 ; +void guestInstClass::readInst ( void ) +{ + int rc ; + std::list socks ; + + waitd.tv_sec = 0; + waitd.tv_usec = GUEST_SOCKET_TO; + + /* Initialize the master fd_set */ + FD_ZERO(&instance_readfds); + + socks.clear(); + + for ( struct inst * inst_ptr = inst_head ; inst_ptr != NULL ; inst_ptr = inst_ptr->next ) + { + if ( inst_ptr->instance.connected ) + { + socks.push_front( inst_ptr->instance.chan_fd ); + FD_SET(inst_ptr->instance.chan_fd, &instance_readfds); + } + if (( inst_ptr->next == NULL ) || ( inst_ptr == inst_tail )) + break ; + } + + /* if there are no connected instance channels then exit */ + if ( socks.empty() ) + { + return ; + } + + /* Call select() and wait only up to SOCKET_WAIT */ + socks.sort(); + rc = select( socks.back()+1, &instance_readfds, NULL, NULL, &waitd); + + if (( rc <= 0 ) || ( rc > (int)socks.size())) + { + /* Check to see if the select call failed. */ + if ( rc > (int)socks.size()) + { + wlog_throttled ( fail_count, 100, "select return exceeds current file descriptors (%ld:%d)\n", + socks.size(), rc ); + } + /* ... but filter Interrupt signal */ + else if (( rc < 0 ) && ( errno != EINTR )) + { + wlog_throttled ( fail_count, 100, "socket select failed (%d:%m)\n", errno); + } + else + { + mlog3 ("nothing received from %ld instances; socket timeout (%d:%m)\n", socks.size(), errno ); + } + } + else + { + fail_count = 0 ; + mlog2 ("trying to receive for %ld instances\n", socks.size()); + + /* Search through all the instances for watched channels */ + for ( struct inst * inst_ptr = inst_head ; inst_ptr != NULL ; inst_ptr = inst_ptr->next ) + { + mlog2 ("%s monitoring %d\n", inst_ptr->instance.inst.c_str(), + inst_ptr->instance.chan_fd ); + + /* Service guestServer messages towards the local IP */ + if (FD_ISSET(inst_ptr->instance.chan_fd, &instance_readfds) ) + { + char buf[GUEST_HEARTBEAT_MSG_MAX_MSG_SIZE] ; + string name ; + + if( inst_ptr->instance.inst.empty() ) + name = inst_ptr->instance.uuid ; + else + name = inst_ptr->instance.inst ; + + struct json_tokener* tok = json_tokener_new(); + + for ( int i = 0; i < INST_MSG_READ_COUNT; i++ ) + { + rc = read ( inst_ptr->instance.chan_fd, buf, GUEST_HEARTBEAT_MSG_MAX_MSG_SIZE); + mlog2 ("%s read channel: bytes:%d, fd:%d\n", name.c_str(), rc,inst_ptr->instance.chan_fd ); + if ( rc < 0 ) + { + if ( errno == EINTR ) + { + wlog_throttled ( inst_ptr->instance.failure_count, 100, "%s EINTR\n", name.c_str()); + } + else if ( errno == ECONNRESET ) + { + wlog ("%s connection reset ... closing\n", name.c_str()); + + /* Close the connection if we get a 'connection reset by peer' errno */ + guestUtil_close_channel ( &inst_ptr->instance ); + + /* An element of the list is removed - need to break out */ + } + else if ( errno != EAGAIN ) + { + wlog_throttled ( inst_ptr->instance.failure_count, 100, "%s error (%d:%m)\n", name.c_str(), errno ); + } + else + { + mlog3 ("%s no more messages\n", name.c_str()); + } + break ; + } + else if ( rc == 0 ) + { + mlog3 ("%s no message\n" , name.c_str()); + break ; + } + else + { + if ( rc < GUEST_HEARTBEAT_MSG_MIN_MSG_SIZE ) + { + wlog_throttled ( inst_ptr->instance.failure_count, 100, + "%s message size %d is smaller than minimal %d; dropping\n", + name.c_str(), rc, GUEST_HEARTBEAT_MSG_MIN_MSG_SIZE); + } + else if ( inst_ptr->message_list.size() > MAX_MESSAGES ) + { + wlog_throttled ( inst_ptr->instance.failure_count, 100, + "%s message queue overflow (max:%d) ; dropping\n", + name.c_str(), MAX_MESSAGES ); + } + else + { + inst_ptr->instance.failure_count = 0 ; + mlog2 ("%s handling message buf: %s\n", name.c_str(), buf ); + handle_virtio_serial_msg(buf, rc, tok, inst_ptr); + } + } + } + json_tokener_free(tok); + } + if (( inst_ptr->next == NULL ) || ( inst_ptr == inst_tail )) + break ; + } + } +} diff --git a/mtce-guest/src/guestSvrMsg.h b/mtce-guest/src/guestSvrMsg.h new file mode 100644 index 00000000..9e61246d --- /dev/null +++ b/mtce-guest/src/guestSvrMsg.h @@ -0,0 +1,27 @@ +#ifndef __INCLUDE_GUESTSVRMSG_H__ +#define __INCLUDE_GUESTSVRMSG_H__ + +/* + * Copyright (c) 2013-2016 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + + /** + * @file + * Wind River CGTS Platform Guest Services "Messaging" Header + */ + +#include "guestBase.h" +#include "guestInstClass.h" /* for ... */ + +/* Send a command and buffer to the guestAgent */ +int send_to_guestAgent ( unsigned int cmd, + const char * buf_ptr ); + +int recv_from_guestAgent ( unsigned int cmd, char * buf_ptr ); + +string guestSvrMsg_hdr_init (string channel, string msg_type); + +#endif /* __INCLUDE_GUESTSVRMSG_H__ */ diff --git a/mtce-guest/src/guestSvrUtil.cpp b/mtce-guest/src/guestSvrUtil.cpp new file mode 100644 index 00000000..4ab15676 --- /dev/null +++ b/mtce-guest/src/guestSvrUtil.cpp @@ -0,0 +1,234 @@ +/* + * Copyright (c) 2015 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +#include "nodeBase.h" /* for ... */ +#include "nodeEvent.h" /* for ... inotify_event_queue_type and utils */ +#include "nodeTimers.h" /* maintenance timer utilities start/stop */ + +#include "guestInstClass.h" +#include "guestUtil.h" /* for ... guestUtil_inst_init */ +#include "guestSvrUtil.h" /* for ... this module header */ +#include "guestVirtio.h" /* for ... virtio_check_filename, + virtio_channel_add */ + +/***************************************************************************** + * + * Name : guestUtil_close_channel + * + * Purpose: Close the specified channel's virtio channel file descriptor. + * + ******************************************************************************/ +int guestUtil_close_channel ( instInfo * instInfo_ptr ) +{ + int rc = FAIL_NOT_FOUND ; + if ( instInfo_ptr ) + { + /* Free up the inotify watch */ + if ( instInfo_ptr->inotify_file_fd ) + { + dlog ("%s freeing inotify resource\n", log_prefix(instInfo_ptr).c_str() ); + set_inotify_close (instInfo_ptr->inotify_file_fd , + instInfo_ptr->inotify_file_wd ); + } + + if ( instInfo_ptr->chan_fd ) + { + dlog ("%s closing socket %d\n", + log_prefix(instInfo_ptr).c_str(), + instInfo_ptr->chan_fd ); + + close ( instInfo_ptr->chan_fd ); + instInfo_ptr->chan_fd = 0 ; + } + instInfo_ptr->chan_ok = false ; + instInfo_ptr->heartbeating = false ; + instInfo_ptr->connected = false ; + rc = PASS ; + } + return (rc); +} + +/***************************************************************************** + * + * Name : guestUtil_load_channels + * + * Purpose: Scan the Virtio Qemu directory looking for heartbeat channels + * into guests. + * + * Load those that are found into the control structure + * and setup messaging to them. + * + ******************************************************************************/ +void guestUtil_load_channels ( void ) +{ + DIR *dirp; + struct dirent entry; + struct dirent *result; + + dirp = opendir(QEMU_CHANNEL_DIR); + if (!dirp) + { + elog("failed to open %s directory (%d:%m)\n", QEMU_CHANNEL_DIR, errno); + } + else + { + dlog ("Searching %s directory\n", QEMU_CHANNEL_DIR); + while(0 == readdir_r(dirp, &entry, &result)) + { + if (!result) + break; + + if ( virtio_check_filename (result->d_name) ) + { + string channel = result->d_name ; + ilog ("%s found\n", channel.c_str() ); + if ( virtio_channel_add ( result->d_name ) == PASS ) + { + if ( virtio_channel_connect ( channel ) != PASS ) + { + string uuid = virtio_instance_name ( result->d_name ) ; + get_instInv_ptr()->reconnect_start ( uuid.data() ); + } + } + } + else + { + dlog3 ("ignoring file %s\n", result->d_name); + } + } + closedir(dirp); + } +} + +/***************************************************************************** + * + * Name : guestUtil_channel_search + * + * Purpose: Scan the Virtio Qemu directory looking for heartbeat channels + * into guests that are not currently provisioned. + * + ******************************************************************************/ +void guestUtil_channel_search ( void ) +{ + DIR *dirp; + struct dirent entry; + struct dirent *result; + + dirp = opendir(QEMU_CHANNEL_DIR); + if (!dirp) + { + elog("failed to open %s directory (%d:%m)\n", QEMU_CHANNEL_DIR, errno); + } + else + { + dlog ("Searching %s directory\n", QEMU_CHANNEL_DIR); + while(0 == readdir_r(dirp, &entry, &result)) + { + if (!result) + break; + + if ( virtio_check_filename (result->d_name) ) + { + if ( get_instInv_ptr()->get_inst ( virtio_instance_name (result->d_name).data()) == NULL ) + { + string channel = result->d_name ; + ilog ("found %s\n", channel.c_str() ); + virtio_channel_add ( result->d_name ); + virtio_channel_connect ( channel ); + } + } + } + closedir(dirp); + } +} + +/***************************************************************************** + * + * Name : guestUtil_inotify_events + * + * Purpose: Handle inotify events for the specified file descriptor. + * + *****************************************************************************/ +int guestUtil_inotify_events ( int fd ) +{ + string channel = "" ; + inotify_event_queue_type event_queue ; + int num = get_inotify_events ( fd , event_queue ) ; + + dlog3 ("inotify events queued: %d\n", num ); + + for ( int i = 0 ; i < num ; i++ ) + { + dlog2 ( "Event:%s for file:%s\n", get_inotify_event_str(event_queue.item[i].event), event_queue.item[i].name ); + + if ( event_queue.item[i].event == IN_CREATE ) + { + dlog1 ("%s CREATE event on %s\n", event_queue.item[i].name, QEMU_CHANNEL_DIR ); + if ( virtio_check_filename (&event_queue.item[i].name[0]) ) + { + dlog ("%s CREATE accepted\n", event_queue.item[i].name ); + channel = event_queue.item[i].name ; + if ( virtio_channel_add ( event_queue.item[i].name ) != PASS ) + { + elog ("%s failed to add detected channel\n", event_queue.item[i].name ); + } + } + } + else if ( event_queue.item[i].event == IN_DELETE ) + { + dlog1 ("%s DELETE event on %s\n", event_queue.item[i].name, QEMU_CHANNEL_DIR ); + if ( virtio_check_filename (&event_queue.item[i].name[0]) ) + { + dlog ("%s DELETE accepted\n", event_queue.item[i].name ); + channel = event_queue.item[i].name ; + get_instInv_ptr()->del_inst ( channel ); + } + else + { + dlog ("%s DELETE rejected\n", event_queue.item[i].name ); + } + } + else if ( event_queue.item[i].event == IN_MODIFY ) + { + dlog1 ("%s MODIFY event on %s\n", event_queue.item[i].name, QEMU_CHANNEL_DIR ); + if ( virtio_check_filename (&event_queue.item[i].name[0]) ) + { + dlog ("%s MODIFY accepted\n", event_queue.item[i].name ); + channel = event_queue.item[i].name ; + + /* if the channel was modified then we need + * + * 1. to close the channel, + * 2. delete it, + * 3. re-add it and + * 4. then repoen it. + * */ + get_instInv_ptr()->del_inst ( channel ); + + if ( virtio_channel_add ( event_queue.item[i].name ) != PASS ) + { + elog ("%s failed to re-add modified channel\n", channel.c_str()); + } + } + } + else + { + wlog ("%s UNKNOWN event on %s\n", event_queue.item[i].name, QEMU_CHANNEL_DIR ); + } + } + return (PASS); +} diff --git a/mtce-guest/src/guestSvrUtil.h b/mtce-guest/src/guestSvrUtil.h new file mode 100644 index 00000000..01b79c24 --- /dev/null +++ b/mtce-guest/src/guestSvrUtil.h @@ -0,0 +1,18 @@ +#ifndef __INCLUDE_GUESTSVRUTIL_H__ +#define __INCLUDE_GUESTSVRUTIL_H__ + +/* + * Copyright (c) 2015 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + +#include "guestBase.h" /* for ... instInfo */ + +int guestUtil_close_channel ( instInfo * instInfo_ptr ); +void guestUtil_load_channels ( void ); +int guestUtil_inotify_events ( int fd ); +void guestUtil_channel_search ( void ) ; + +#endif /* __INCLUDE_GUESTSVRUTIL_H__ */ diff --git a/mtce-guest/src/guestUtil.cpp b/mtce-guest/src/guestUtil.cpp new file mode 100644 index 00000000..6b72fc03 --- /dev/null +++ b/mtce-guest/src/guestUtil.cpp @@ -0,0 +1,330 @@ +/* + * Copyright (c) 2013-2016 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + +#include +#include +#include +using namespace std; + +#include "guestBase.h" +#include "guestUtil.h" +#include "guestClass.h" +#include "jsonUtil.h" + +#define MAX_NUM_LEN 64 +string time_in_secs_to_str ( time_t secs ) +{ + char int_str[MAX_NUM_LEN] ; + string temp ; + memset ( &int_str[0], 0, MAX_NUM_LEN ); + sprintf ( &int_str[0], "%ld" , secs ); + temp = int_str ; + return (temp); +} + +/***************************************************************************** + * + * Name : guestUtil_inst_init + * + * Purpose: Init the specified instance + * + *****************************************************************************/ +void guestUtil_inst_init ( instInfo * instance_ptr ) +{ + instance_ptr->uuid.clear(); /* Not used in the server */ + + instance_ptr->inotify_file_fd = 0 ; + instance_ptr->inotify_file_wd = 0 ; + + instance_ptr->chan_fd = 0 ; + instance_ptr->chan_ok = false ; + instance_ptr->connected = false ; /* Assume we have not connected to this channel */ + instance_ptr->heartbeating = false ; + + instance_ptr->heartbeat.provisioned = false ; + instance_ptr->heartbeat.reporting = false ; + instance_ptr->heartbeat.failures = 0 ; + + instance_ptr->heartbeat.state.clear() ; + + instance_ptr->hbState = hbs_server_waiting_init ; + instance_ptr->vnState = hbs_server_waiting_init ; + + instance_ptr->connect_count = 0 ; + instance_ptr->connect_retry_count = 0 ; + instance_ptr->select_count = 0 ; + instance_ptr->message_count = 0 ; + instance_ptr->health_count = 0 ; + instance_ptr->failure_count = 0 ; + instance_ptr->corrective_action_count = 0 ; + + instance_ptr->unhealthy_failure = false ; + + instance_ptr->heartbeat_interval_ms = HB_DEFAULT_INTERVAL_MS; + + instance_ptr->vote_secs = HB_DEFAULT_VOTE_MS/1000; + instance_ptr->vote_to_str = time_in_secs_to_str (instance_ptr->vote_secs); + + instance_ptr->shutdown_notice_secs = HB_DEFAULT_SHUTDOWN_MS/1000; + instance_ptr->shutdown_to_str = time_in_secs_to_str (instance_ptr->shutdown_notice_secs); + + instance_ptr->suspend_notice_secs = HB_DEFAULT_SUSPEND_MS/1000; + instance_ptr->suspend_to_str = time_in_secs_to_str (instance_ptr->suspend_notice_secs); + + instance_ptr->resume_notice_secs = HB_DEFAULT_RESUME_MS/1000; + instance_ptr->resume_to_str = time_in_secs_to_str (instance_ptr->resume_notice_secs); + + instance_ptr->restart_secs = HB_DEFAULT_RESTART_MS/1000; + instance_ptr->restart_to_str = time_in_secs_to_str(instance_ptr->restart_secs); + + instance_ptr->notification_type = GUEST_HEARTBEAT_MSG_NOTIFY_IRREVOCABLE ; + instance_ptr->event_type = GUEST_HEARTBEAT_MSG_EVENT_RESUME ; + + instance_ptr->corrective_action = GUEST_HEARTBEAT_MSG_ACTION_LOG ; + + instance_ptr->unhealthy_corrective_action = GUEST_HEARTBEAT_MSG_ACTION_UNKNOWN ; +} + +/***************************************************************************** + * + * Name : guestUtil_print_instance + * + * Purpose: Print a summary of the instances that are currently provisioned + * + *****************************************************************************/ +void guestUtil_print_instance ( instInfo * instInfo_ptr ) +{ + ilog ("%s Heartbeat: Prov-%c Reporting-%c Failures:%d\n", + instInfo_ptr->uuid.c_str(), + instInfo_ptr->heartbeat.provisioned ? 'Y':'n' , + instInfo_ptr->heartbeat.reporting ? 'Y':'n', + instInfo_ptr->heartbeat.failures); +} + +/***************************************************************************** + * + * Name : guestUtil_print_instances + * + * Purpose: Print a summary of the instances that are currently provisioned + * + *****************************************************************************/ +void guestUtil_print_instances ( ctrl_type * ctrl_ptr ) +{ + bool found = false ; + int i = 1 ; + + for ( ctrl_ptr->instance_list_ptr = ctrl_ptr->instance_list.begin(); + ctrl_ptr->instance_list_ptr != ctrl_ptr->instance_list.end(); + ctrl_ptr->instance_list_ptr++ ) + { + guestUtil_print_instance ( &(*ctrl_ptr->instance_list_ptr) ); + found = true ; + i++ ; + } + + if ( found == false ) + { + ilog ("no heartbeat channels provisioned\n"); + } +} + +string log_prefix ( instInfo * instInfo_ptr ) +{ + string prefix = "unknown" ; + + if ( instInfo_ptr ) + { + if ( instInfo_ptr->name.length() ) + { + if ( instInfo_ptr->name_log_prefix.empty() ) + { + instInfo_ptr->name_log_prefix = instInfo_ptr->inst ; + instInfo_ptr->name_log_prefix.append (" "); + instInfo_ptr->name_log_prefix.append (instInfo_ptr->name); + } + prefix = instInfo_ptr->name_log_prefix ; + } + else + { + if ( instInfo_ptr->uuid_log_prefix.empty() ) + { + instInfo_ptr->uuid_log_prefix = instInfo_ptr->uuid ; + } + prefix = instInfo_ptr->uuid_log_prefix ; + } + } + return (prefix); +} + +string guestUtil_set_inst_info ( string hostname , instInfo * instInfo_ptr ) +{ + /* Send one message per instance */ + string payload ("{\"hostname\":\""); + payload.append (hostname); + payload.append ("\",\"uuid\":\""); + payload.append (instInfo_ptr->uuid); + + /* Share the reporting state */ + payload.append ("\",\"reporting\":"); + if ( instInfo_ptr->heartbeat.reporting == true ) + payload.append ("\"enabled"); + else + payload.append ("\"disabled"); + + /* Share the heartbeating state */ + payload.append ("\",\"heartbeating\":"); + if ( instInfo_ptr->heartbeating == true ) + payload.append ("\"enabled"); + else + payload.append ("\"disabled"); + + payload.append ("\",\"repair-action\":\"" ); + if ( instInfo_ptr->unhealthy_failure == true ) + { + payload.append (instInfo_ptr->unhealthy_corrective_action); + } + else + { + payload.append (instInfo_ptr->corrective_action); + } + /* Add the restart timeout to the message */ + payload.append ("\",\"restart-to\":\""); + payload.append (instInfo_ptr->restart_to_str); + payload.append ("\",\"shutdown-to\":\""); + payload.append (instInfo_ptr->shutdown_to_str); + payload.append ("\",\"suspend-to\":\""); + payload.append (instInfo_ptr->suspend_to_str); + payload.append ("\",\"resume-to\":\""); + payload.append (instInfo_ptr->resume_to_str); + payload.append ("\",\"vote-to\":\""); + payload.append (instInfo_ptr->vote_to_str); + payload.append ("\""); + payload.append ("}"); + + jlog ("Payload: %s\n", payload.c_str()); + + return (payload); +} + +int guestUtil_get_inst_info ( string hostname, instInfo * instInfo_ptr, char * buf_ptr ) +{ + int rc = PASS ; + + string hostname_str = "" ; + string uuid = "" ; + string state = "" ; + string status = "" ; + string restart_to = "" ; + string resume_to = "" ; + string suspend_to = "" ; + string shutdown_to = "" ; + string vote_to = "" ; + string repair_str = "" ; + + if ( !buf_ptr ) + { + elog ( "null buffer\n" ); + return ( FAIL_NULL_POINTER ); + } + + jlog ("Payload: %s\n", buf_ptr ); + + int rc0 = jsonUtil_get_key_val ( buf_ptr, "hostname", hostname_str) ; + int rc1 = jsonUtil_get_key_val ( buf_ptr, "uuid", uuid ) ; + int rc2 = jsonUtil_get_key_val ( buf_ptr, "reporting", state ) ; + int rc3 = jsonUtil_get_key_val ( buf_ptr, "heartbeating", status ) ; + int rc4 = jsonUtil_get_key_val ( buf_ptr, "restart-to", restart_to ) ; + int rc5 = jsonUtil_get_key_val ( buf_ptr, "resume-to", resume_to ) ; + int rc6 = jsonUtil_get_key_val ( buf_ptr, "suspend-to", suspend_to ) ; + int rc7 = jsonUtil_get_key_val ( buf_ptr, "shutdown-to", shutdown_to ) ; + int rc8 = jsonUtil_get_key_val ( buf_ptr, "vote-to", vote_to ) ; + int rc9= jsonUtil_get_key_val ( buf_ptr, "repair-action",repair_str ) ; + if ( rc0 | rc1 | rc2 | rc3 | rc4 | rc5 | rc6 | rc7 | rc8 | rc9 ) + { + elog ("%s failed parse one or more key values (%d:%d:%d:%d:%d:%d:%d:%d:%d:%d)\n", + hostname.c_str(), rc0, rc1, rc2, rc3, rc4, rc5, rc6, rc7, rc8, rc9); + + rc = FAIL_KEY_VALUE_PARSE ; + } + else + { + if ( hostname.compare(hostname_str) ) + { + wlog ("%s hostname mismatch - loaded\n", hostname_str.c_str()); + } + + if ( instInfo_ptr ) + { + /* Update the reporting state */ + if ( !state.compare("enabled") ) + instInfo_ptr->heartbeat.reporting = true ; + else + instInfo_ptr->heartbeat.reporting = false ; + + /* update the heartbeating status */ + if ( !status.compare("enabled") ) + instInfo_ptr->heartbeating = true ; + else + instInfo_ptr->heartbeating = false ; + + instInfo_ptr->corrective_action = repair_str ; + + /* Update the intance timeout values */ + instInfo_ptr->restart_to_str = restart_to ; + instInfo_ptr->shutdown_to_str = shutdown_to ; + instInfo_ptr->resume_to_str = resume_to ; + instInfo_ptr->suspend_to_str = suspend_to ; + instInfo_ptr->vote_to_str = vote_to ; + } + else + { + wlog ("%s %s lookup failed\n", hostname.c_str(), uuid.c_str()); + rc = FAIL_INVALID_UUID ; + } + } + return (rc); +} + +const char* state_names[] = + { + "invalid", + "server_waiting_init", + "server_waiting_challenge", + "server_waiting_response", + "server_paused", + "server_nova_paused", + "server_migrating", + "server_corrective_action", + "client_waiting_init_ack", + "client_waiting_challenge", + "client_waiting_pause_ack", + "client_waiting_resume_ack", + "client_paused", + "client_waiting_shutdown_ack", + "client_waiting_shutdown_response", + "client_shutdown_response_recieved", + "client_exiting", + }; + +const char* hb_get_state_name ( hb_state_t s ) +{ + if (s >= hbs_state_max) + return "???"; + + return state_names[s]; +} + +/***************************************************************************** + * Convert integer to string + *****************************************************************************/ +string int_to_string(int number) +{ + ostringstream ostr; + ostr << number; + return ostr.str(); +} + diff --git a/mtce-guest/src/guestUtil.h b/mtce-guest/src/guestUtil.h new file mode 100644 index 00000000..f1926f53 --- /dev/null +++ b/mtce-guest/src/guestUtil.h @@ -0,0 +1,34 @@ +#ifndef __INCLUDE_GUESTUTIL_H__ +#define __INCLUDE_GUESTUTIL_H__ + +/* + * Copyright (c) 2015 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + +#include "guestBase.h" /* for ... instInfo */ + +void guestUtil_inst_init ( instInfo * instance_ptr ); +void guestUtil_print_instances ( ctrl_type * ctrl_ptr ); +void guestUtil_print_instance ( instInfo * instInfo_ptr ); + +/* called in guestAgent */ +int guestUtil_get_inst_info ( string hostname, instInfo * instInfo_ptr, char * buf_ptr ); + +/* called in guestServer */ +string guestUtil_set_inst_info ( string hostname, instInfo * instInfo_ptr ); + + + +string log_prefix ( instInfo * instInfo_ptr ); +string time_in_secs_to_str ( time_t secs ); + +const char* hb_get_corrective_action_name( uint32_t a) ; // heartbeat_corrective_action_t a); +const char* hb_get_state_name (hb_state_t s); +// Convert integer to string +string int_to_string(int number); + + +#endif /* __INCLUDE_GUESTUTIL_H__ */ diff --git a/mtce-guest/src/guestVimApi.cpp b/mtce-guest/src/guestVimApi.cpp new file mode 100644 index 00000000..369ff9e3 --- /dev/null +++ b/mtce-guest/src/guestVimApi.cpp @@ -0,0 +1,777 @@ +/* + * Copyright (c) 2013, 2016 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + + /** + * @file Wind River CGTS Platform Guest Heartbeat REST API + * used to report heartbeat faults or query instance + * information from the VIM. + * + */ + +#ifdef __AREA__ +#undef __AREA__ +#endif +#define __AREA__ "vim" + +using namespace std; + +#include "nodeBase.h" /* for ... common definitions */ +#include "nodeUtil.h" /* for ... common utilities */ +#include "jsonUtil.h" /* for ... jsonUtil_get_key_val */ + +#include "guestUtil.h" /* for ... guestUtil_inst_init */ +#include "guestSvrUtil.h" /* for ... hb_get_corrective_action_name */ +#include "guestVimApi.h" /* for ... this module header */ + +#define URL_VIM_ADDRESS "127.0.0.1" +#define URL_VIM_INST_LABEL "/nfvi-plugins/v1/instances/" +#define URL_VIM_HOST_LABEL "/nfvi-plugins/v1/hosts/" + +#define VIM_EVENT_SIG "vimEvent" +#define VIM_SIG "vim" + +#define OPER__HOST_STATE_QUERY "host state query" +#define OPER__HOST_INST_QUERY "host inst query" +#define OPER__HOST_INST_FAIL "host inst fail" +#define OPER__HOST_INST_STATUS "host inst status" +#define OPER__HOST_INST_CHANGE "inst status change" +#define OPER__HOST_INST_NOTIFY "host inst notify" + + +/********************************************************************* + * + * Name : guestVimApi_handler + * + * Description: The Guest Heartbeat event request handler + * + *********************************************************************/ +void guestHostClass::guestVimApi_handler ( struct evhttp_request *req, void *arg ) +{ + string hostname = "unknown" ; + + guestHostClass * obj_ptr = get_hostInv_ptr(); + libEvent & event = obj_ptr->getEvent ( (struct event_base *)arg, hostname ); + if ( event.request == SERVICE_NONE ) + { + slog ("guest instance Lookup Failed (%p)\n", arg); + return ; + } + + /* Check for command timeout */ + if ( !req ) + { + dlog ("hostname=%s service=%s No Request Parm (%s)\n", + event.hostname.c_str(), + event.service.c_str(), + event.uuid.c_str()); + } + + /* Check the HTTP Status Code */ + event.status = guestHttpUtil_status ( event ) ; + if ( event.status == HTTP_NOTFOUND ) + { + wlog ("%s Not Found (%d)\n", event.log_prefix.c_str(), + event.status); + goto _guest_handler_done ; + } + + else if ( event.status != PASS ) + { + /* The VIM Might not be running at he time I issue the query. + * In hat case I will get back a 400 */ + if (( event.request != VIM_HOST_STATE_QUERY ) && ( event.status != 400 )) + { + elog ("%s HTTP Request Failed (%d) (%s)\n", + event.log_prefix.c_str(), + event.status, + event.uuid.c_str()); + } + goto _guest_handler_done ; + } + + /* No response content for this command */ + if ( event.request == VIM_HOST_INSTANCE_STATUS ) + { + jlog ("%s %s instance status change succeeded\n", event.hostname.c_str(), event.uuid.c_str()); + goto _guest_handler_done ; + } + + /* No response content for this command */ + else if ( event.request == VIM_HOST_INSTANCE_NOTIFY ) + { + jlog ("%s %s instance notify succeeded\n", event.hostname.c_str(), event.uuid.c_str()); + goto _guest_handler_done ; + } + + else if ( httpUtil_get_response ( event ) != PASS ) + { + wlog ("%s no response available\n", hostname.c_str()); + goto _guest_handler_done ; + } + + if ( event.response.length() ) + { + jlog ("%s Response: %s\n", event.hostname.c_str(), + event.response.c_str()); + + if ( event.request == VIM_HOST_STATE_QUERY ) + { + ilog ("%s host state query response\n", event.hostname.c_str()); + int rc = jsonUtil_get_key_val ( (char*)event.response.data(), "state", event.value ) ; + if ( rc != PASS ) + { + elog ("failed to state value (rc=%d)\n", rc ); + event.status = FAIL_KEY_VALUE_PARSE ; + event.value = "disabled" ; /* override to disabled if operation failed */ + } + } + + else if ( event.request == VIM_HOST_INSTANCE_FAILED ) + { + ilog ("%s instance failure response\n", event.uuid.c_str()); + // {"services": [ {"state": "enabled", "service": "heartbeat"}], + // "hostname": "compute-1", + // "uuid": "da973c2a-7469-4e06-b7e1-89bf2643f906"} + string state = "" ; + string service = "" ; + string uuid = "" ; + int rc1 = jsonUtil_get_key_val ( (char*)event.response.data(), "hostname", hostname ) ; + int rc2 = jsonUtil_get_key_val ( (char*)event.response.data(), "uuid" , uuid ) ; + if (!(rc1 | rc2 )) + { + /* Look for the list of services for this instance + * - currently only heartbeat is supported + * + * services:[ { "state": "enabled", "service": "heartbeat" } ] + */ + string service_list = "" ; + rc1 = jsonUtil_get_array_idx ((char*)event.response.data(), "services", 0, service_list ) ; + if ( rc1 == PASS ) + { + instInfo instance ; guestUtil_inst_init ( &instance ); + guestHostClass * obj_ptr = get_hostInv_ptr(); + string service = "" ; + + ilog ("Service List:%s\n", service_list.c_str()); // jlog1 + + instance.uuid = uuid ; + + /* Get the contents of the services list/array + * Note: we only support one element of the array so hat's + * why only index 0 is being requested or looked for + * + * Get the state of the only service - heartbeat */ + rc1 = jsonUtil_get_key_val ( (char*)service_list.data(), "state", instance.heartbeat.state ) ; + rc2 = jsonUtil_get_key_val ( (char*)service_list.data(), "service", service ) ; + + /* both of these must pass in order to add this instance */ + if (( rc1 == PASS ) && ( rc2 == PASS )) + { + if ( !service.compare("heartbeat") ) + { + instance.heartbeat.provisioned = true ; + + /* Its either enabled or disabled + * - default was disabled in guestUtil_inst_init above */ + if ( !instance.heartbeat.state.compare("enabled") ) + { + instance.heartbeat.reporting = true ; + rc1 = obj_ptr->mod_inst ( hostname, instance ); + } + else if ( !instance.heartbeat.state.compare("disabled") ) + { + instance.heartbeat.reporting = false ; + rc1 = obj_ptr->mod_inst ( hostname, instance ); + } + else + { + // raise error if it is neither enabled nor disabled + elog ("%s %s invalid heartbeat.state value %s received\n", + hostname.c_str(), instance.uuid.c_str(), instance.heartbeat.state.c_str()); + event.status = FAIL_INVALID_DATA ; + rc1 = FAIL; + } + if ( rc1 == PASS ) + { + /* o.K. so its provisioned !! */ + dlog ("%s %s instance modified\n", hostname.c_str(), instance.uuid.c_str()); + } + else + { + event.status = rc1 ; + } + } + else + { + elog ("%s unsupported 'service' (%s)\n", hostname.c_str(), service.c_str() ); + event.status = FAIL_INVALID_DATA ; + } + } + else + { + elog ("%s failed to get 'state' or 'service' (%d:%d)\n", hostname.c_str(), rc1, rc2 ); + event.status = FAIL_KEY_VALUE_PARSE ; + } + } + else + { + elog ("%s failed to get 'service list' or 'uuid' (%d:%d)\n", hostname.c_str(), rc1, rc2 ); + event.status = FAIL_KEY_VALUE_PARSE ; + } + } + else + { + ilog ("%s failed to get 'hostname' or 'uuid' (%d:%d)\n", event.hostname.c_str(), rc1, rc2 ); + event.status = FAIL_KEY_VALUE_PARSE ; + } + } + else if ( event.request == VIM_HOST_INSTANCE_QUERY ) + { + ilog ("%s instance query response\n", event.uuid.c_str()); + /* { "instances": [{"services": {"service":"heartbeat", "state":"enabled"}, + * "hostname": "compute-2", + * "uuid": "3aca8dad-0e38-4a58-83ab-23ee71159e0d"}]} */ + + int rc = jsonUtil_get_key_val ( (char*)event.response.data(), "instances", event.value ) ; + if ( rc != PASS ) + { + elog ("%s failed to get host instance array (rc=%d) (%s)\n", + event.hostname.c_str(), rc, event.uuid.c_str()); + event.status = FAIL_KEY_VALUE_PARSE ; + } + else + { + /* The following code parses a JSON string that looks like this. + * { + * "instances": + * [ + * { "services": { "service":"heartbeat", "state":"enabled" }, + * "hostname": "compute-2", + * "uuid" : "3aca8dad-0e38-4a58-83ab-23ee71159e0d" + * } + * ] , ... + * } + */ + int instances = 0 ; + jlog ("%s instance array %s\n", event.hostname.c_str(), (char*)event.response.data()); + rc = jsonUtil_array_elements ( (char*)event.response.data(), "instances", instances ); + if ( rc != PASS ) + { + elog ("%s failed to get array elements (%d)\n", hostname.c_str(), rc ); + event.status = FAIL_KEY_VALUE_PARSE ; + } + else + { + ilog ("%s has %d instances\n", hostname.c_str(), instances ); + for ( int i = 0 ; i < instances ; i++ ) + { + string instance_element = "" ; + rc = jsonUtil_get_array_idx ( (char*)event.response.data(), "instances", i, instance_element ); + if ( ( rc == PASS ) && ( instance_element.size() )) + { + /* Look for the list of services for this instance + * - currently only heartbeat is supported + * + * services:[ { "state": "enabled", "service": "heartbeat" } ] + **/ + string service_list = "" ; + string uuid = "" ; + int rc1 = jsonUtil_get_array_idx ((char*)instance_element.data(), "services", 0, service_list ) ; + int rc2 = jsonUtil_get_key_val ((char*)instance_element.data(), "uuid", uuid ) ; + if (( rc1 == PASS ) && ( rc2 == PASS )) + { + instInfo instance ; guestUtil_inst_init ( &instance ); + guestHostClass * obj_ptr = get_hostInv_ptr(); + string service = "" ; + + ilog ("Service List:%s\n", service_list.c_str()); + + instance.uuid = uuid ; + + /* Get the contents of the services list/array + * Note: we only support one element of the array so hat's + * why only index 0 is being requested or looked for + * + * Get the state of the only service - heartbeat */ + rc1 = jsonUtil_get_key_val ( (char*)service_list.data(), "state", instance.heartbeat.state ) ; + rc2 = jsonUtil_get_key_val ( (char*)service_list.data(), "service", service ) ; + + /* both of these must pass in order to add this instance */ + if (( rc1 == PASS ) && ( rc2 == PASS )) + { + if ( !service.compare("heartbeat") ) + { + instance.heartbeat.provisioned = true ; + + /* Its either enabled or disabled + * - default was disabled in guestUtil_inst_init above */ + if ( !instance.heartbeat.state.compare("enabled") ) + { + instance.heartbeat.reporting = true ; + rc = obj_ptr->add_inst ( hostname, instance ); + } + else if ( !instance.heartbeat.state.compare("disabled") ) + { + instance.heartbeat.reporting = false ; + rc = obj_ptr->add_inst ( hostname, instance ); + } + else + { + // raise error if it is neither enabled nor disabled + elog ("%s %s invalid heartbeat.state value %s received\n", + hostname.c_str(), instance.uuid.c_str(), instance.heartbeat.state.c_str()); + event.status = FAIL_INVALID_DATA ; + rc = FAIL; + } + if ( rc == PASS ) + { + /* o.K. so its provisioned !! */ + ilog ("%s %s instance added\n", hostname.c_str(), instance.uuid.c_str()); + } + else + { + event.status = rc ; + } + } + else + { + elog ("%s unsupported 'service' (%s)\n", hostname.c_str(), service.c_str() ); + event.status = FAIL_INVALID_DATA ; + } + } + else + { + elog ("%s failed to get 'state' or 'service' (%d:%d)\n", hostname.c_str(), rc1, rc2 ); + wlog ("... Service List: %s\n", service_list.data()); + event.status = FAIL_KEY_VALUE_PARSE ; + } + } + else + { + elog ("%s failed to get 'service list' or 'uuid' (%d:%d)\n", hostname.c_str(), rc1, rc2 ); + event.status = FAIL_KEY_VALUE_PARSE ; + } + } + else if ( rc != PASS ) + { + elog ("%s failed to get array index %d (rc=%d)\n", hostname.c_str(), i, rc ); + event.status = FAIL_KEY_VALUE_PARSE ; + } + } + } + } + } + } +_guest_handler_done: + + // httpUtil_log_event ( event ); + + if (( event.request != SERVICE_NONE ) && + ( event.status != HTTP_OK ) && + ( event.status != PASS )) + { + // wlog ("Event Status: %d\n", event.status ); + + /* TODO: Enable log_event */ + wlog ("%s Address : %s (%d)\n", + event.log_prefix.c_str(), + event.address.c_str(), + event.status); + elog ("%s Payload : %s\n", event.log_prefix.c_str(), event.payload.c_str()); + if ( event.response.size() ) + { + elog ("%s Response: %s\n", event.log_prefix.c_str(), event.response.c_str()); + } + else + { + elog ("%s: no response\n", event.log_prefix.c_str()); + } + } + event.active = false ; + httpUtil_free_conn ( event ); + httpUtil_free_base ( event ); + + /* This is needed to get out of the loop */ + event_base_loopbreak((struct event_base *)arg); +} + +/* The Guest Heartbeat event request handler + * wrapper abstracted from guestHostClass */ +void guestVimApi_Handler ( struct evhttp_request *req, void *arg ) +{ + get_hostInv_ptr()->guestVimApi_handler ( req , arg ); +} + +/***************************************************************************** + * + * Name : guestVimApi_svc_event + * + * Description: Send a VM instance service state/status change notification + * to the VIM. + * + * Warning : Only the 'heartbeat' service 'status' change is supported. + * + *****************************************************************************/ + +int guestVimApi_svc_event ( string hostname, + string instance_uuid, + string state, + string status, + string timeout) +{ + guestHostClass * obj_ptr = get_hostInv_ptr() ; + + ilog ("%s %s %s heartbeating status change to '%s' (to vim)\n", hostname.c_str(), + instance_uuid.c_str(), + state.c_str(), + status.c_str()); + + instInfo * instInfo_ptr = obj_ptr->get_inst ( instance_uuid ); + if ( instInfo_ptr ) + { + httpUtil_event_init ( &instInfo_ptr->vimEvent, + hostname, + VIM_SIG, + URL_VIM_ADDRESS, + daemon_get_cfg_ptr()->vim_event_port); + + instInfo_ptr->vimEvent.base = NULL ; + instInfo_ptr->vimEvent.conn = NULL ; + + /* Set the host context */ + instInfo_ptr->vimEvent.uuid = instance_uuid ; + instInfo_ptr->vimEvent.cur_retries = 0 ; + instInfo_ptr->vimEvent.max_retries = 3 ; + instInfo_ptr->vimEvent.active = true ; + instInfo_ptr->vimEvent.noncritical = false ; + + instInfo_ptr->vimEvent.request = VIM_HOST_INSTANCE_STATUS; + instInfo_ptr->vimEvent.operation = OPER__HOST_INST_CHANGE ; + instInfo_ptr->vimEvent.token.url = URL_VIM_INST_LABEL ; + instInfo_ptr->vimEvent.token.url.append(instance_uuid) ; + + /* The type of HTTP request */ + instInfo_ptr->vimEvent.type = EVHTTP_REQ_PATCH ; + + /* Build the payload */ + instInfo_ptr->vimEvent.payload = ("{\"uuid\":\""); + instInfo_ptr->vimEvent.payload.append (instance_uuid); + instInfo_ptr->vimEvent.payload.append ("\",\"hostname\":\""); + instInfo_ptr->vimEvent.payload.append (hostname); + instInfo_ptr->vimEvent.payload.append ("\",\"event-type\":\"service\",\"event-data\":{\"services\":"); + instInfo_ptr->vimEvent.payload.append ("[{\"service\":\"heartbeat\",\"state\":\""); + instInfo_ptr->vimEvent.payload.append (state); + instInfo_ptr->vimEvent.payload.append ("\",\"status\":\""); + instInfo_ptr->vimEvent.payload.append (status); + instInfo_ptr->vimEvent.payload.append ("\",\"restart-timeout\":\""); + instInfo_ptr->vimEvent.payload.append (timeout); + instInfo_ptr->vimEvent.payload.append ("\"}]}}"); + + jlog ("%s %s Payload: %s\n", hostname.c_str(), instance_uuid.c_str(), instInfo_ptr->vimEvent.payload.c_str()); + + return (guestHttpUtil_api_req ( instInfo_ptr->vimEvent )); + } + return (FAIL_HOSTNAME_LOOKUP); +} + + + + +/***************************************************************************** + * + * Name : guestVimApi_alarm_event + * + * Description: Send a VM instance service an alarm event. + * + *****************************************************************************/ + +int guestVimApi_alarm_event ( string hostname, + string instance_uuid) +{ + guestHostClass * obj_ptr = get_hostInv_ptr() ; + + ilog ("%s %s heartbeating alarm (ill health) event (to vim)\n", + hostname.c_str(), + instance_uuid.c_str()); + + instInfo * instInfo_ptr = obj_ptr->get_inst ( instance_uuid ); + if ( instInfo_ptr ) + { + httpUtil_event_init ( &instInfo_ptr->vimEvent, + hostname, + VIM_SIG, + URL_VIM_ADDRESS, + daemon_get_cfg_ptr()->vim_event_port); + + instInfo_ptr->vimEvent.base = NULL ; + instInfo_ptr->vimEvent.conn = NULL ; + + /* Set the host context */ + instInfo_ptr->vimEvent.uuid = instance_uuid ; + instInfo_ptr->vimEvent.cur_retries = 0 ; + instInfo_ptr->vimEvent.max_retries = 3 ; + instInfo_ptr->vimEvent.active = true ; + instInfo_ptr->vimEvent.noncritical = false ; + + instInfo_ptr->vimEvent.request = VIM_HOST_INSTANCE_STATUS; + instInfo_ptr->vimEvent.operation = OPER__HOST_INST_CHANGE ; + instInfo_ptr->vimEvent.token.url = URL_VIM_INST_LABEL ; + instInfo_ptr->vimEvent.token.url.append(instance_uuid) ; + + /* The type of HTTP request */ + instInfo_ptr->vimEvent.type = EVHTTP_REQ_PATCH ; + + /* Build the payload */ + instInfo_ptr->vimEvent.payload = ("{\"uuid\":\""); + instInfo_ptr->vimEvent.payload.append (instance_uuid); + instInfo_ptr->vimEvent.payload.append ("\",\"hostname\":\""); + instInfo_ptr->vimEvent.payload.append (hostname); + + instInfo_ptr->vimEvent.payload.append ("\",\"event-type\":\"alarm\",\"event-data\":{\"services\":"); + instInfo_ptr->vimEvent.payload.append ("[{\"service\":\"heartbeat\",\"state\":\"unhealthy\",\"repair-action\":\""); + instInfo_ptr->vimEvent.payload.append (instInfo_ptr->corrective_action); + instInfo_ptr->vimEvent.payload.append ("\"}]}}"); + + jlog ("%s %s Payload: %s\n", hostname.c_str(), + instance_uuid.c_str(), + instInfo_ptr->vimEvent.payload.c_str()); + + return (guestHttpUtil_api_req ( instInfo_ptr->vimEvent )); + } + return (FAIL_HOSTNAME_LOOKUP); +} + + +/***************************************************************************** + * + * Name : guestVimApi_inst_failed + * + * Description: Send a VM instance a failure notification to the VIM. + * + * Supported failures are ... + * + * MTC_EVENT_HEARTBEAT_LOSS + * + *****************************************************************************/ +int guestVimApi_inst_failed ( string hostname, + string instance_uuid, + unsigned int event, + int retries ) +{ + guestHostClass * obj_ptr = get_hostInv_ptr() ; + + elog ("%s %s *** Heartbeat Loss *** \n", + hostname.c_str(), + instance_uuid.c_str() ); + + if ( obj_ptr->get_reporting_state (hostname) == false ) + { + ilog ("%s cancelling failure notification request\n", hostname.c_str()); + ilog ("%s ... 'host' level fault reporting is disabled\n", hostname.c_str()); + return (PASS); + } + instInfo * instInfo_ptr = obj_ptr->get_inst ( instance_uuid ); + if ( instInfo_ptr ) + { + if (( event == MTC_EVENT_HEARTBEAT_LOSS ) && + ( instInfo_ptr->heartbeat.reporting == false )) + { + ilog ("%s cancelling failure notification request\n", hostname.c_str()); + ilog ("%s ... 'instance' level fault reporting is disabled\n", hostname.c_str()); + return (PASS); + } + + httpUtil_event_init ( &instInfo_ptr->vimEvent, + hostname, + VIM_SIG, + URL_VIM_ADDRESS, + daemon_get_cfg_ptr()->vim_event_port); + + instInfo_ptr->vimEvent.base = NULL ; + instInfo_ptr->vimEvent.conn = NULL ; + + /* Set the host context */ + instInfo_ptr->vimEvent.uuid = instance_uuid ; + instInfo_ptr->vimEvent.cur_retries = 0 ; + instInfo_ptr->vimEvent.max_retries = retries ; + instInfo_ptr->vimEvent.active = true ; + instInfo_ptr->vimEvent.noncritical = false ; + + instInfo_ptr->vimEvent.request = VIM_HOST_INSTANCE_FAILED; + instInfo_ptr->vimEvent.operation = OPER__HOST_INST_FAIL ; + instInfo_ptr->vimEvent.token.url = URL_VIM_INST_LABEL ; + instInfo_ptr->vimEvent.token.url.append(instance_uuid) ; + + /* The type of HTTP request */ + instInfo_ptr->vimEvent.type = EVHTTP_REQ_PATCH ; + + /* Build the payload */ + instInfo_ptr->vimEvent.payload = ("{\"uuid\":\""); + instInfo_ptr->vimEvent.payload.append (instance_uuid); + instInfo_ptr->vimEvent.payload.append ("\",\"hostname\":\""); + instInfo_ptr->vimEvent.payload.append (hostname); + if ( event == MTC_EVENT_HEARTBEAT_LOSS ) + { + instInfo_ptr->vimEvent.payload.append ("\",\"event-type\":\"alarm\",\"event-data\":{\"services\":"); + instInfo_ptr->vimEvent.payload.append ("[{\"service\":\"heartbeat\",\"state\":\"failed\",\"repair-action\":\""); + instInfo_ptr->vimEvent.payload.append (instInfo_ptr->corrective_action); + instInfo_ptr->vimEvent.payload.append ("\"}]}}"); + + wlog ("%s %s Payload: %s\n", hostname.c_str(), + instance_uuid.c_str(), + instInfo_ptr->vimEvent.payload.c_str()); + } + else + { + elog ("%s Unsupported 'event code' (%d)\n", instance_uuid.c_str(), event ); + return (FAIL_BAD_PARM); + } + + return (guestHttpUtil_api_req ( instInfo_ptr->vimEvent )); + } + return (FAIL_HOSTNAME_LOOKUP); +} + + +/***************************************************************************** + * + * Name : guestVimApi_inst_action + * + * Description: Send a notify message to the VIM in response to voting or notification + * + *****************************************************************************/ +int guestVimApi_inst_action ( string hostname, + string instance_uuid, + string action, + string guest_response, + string reason, + int retries) +{ + guestHostClass * obj_ptr = get_hostInv_ptr() ; + + ilog ("%s %s '%s' action (to vim)\n", hostname.c_str(), instance_uuid.c_str() , action.c_str() ); + + instInfo * instInfo_ptr = obj_ptr->get_inst ( instance_uuid ); + if ( !instInfo_ptr ) + return FAIL_HOSTNAME_LOOKUP; + + httpUtil_event_init ( &instInfo_ptr->vimEvent, + hostname, + VIM_SIG, + URL_VIM_ADDRESS, + daemon_get_cfg_ptr()->vim_event_port); + + instInfo_ptr->vimEvent.base = NULL ; + instInfo_ptr->vimEvent.conn = NULL ; + + /* Set the host context */ + instInfo_ptr->vimEvent.uuid = instance_uuid ; + instInfo_ptr->vimEvent.cur_retries = 0 ; + instInfo_ptr->vimEvent.max_retries = retries ; + instInfo_ptr->vimEvent.active = true ; + instInfo_ptr->vimEvent.noncritical = false ; + + instInfo_ptr->vimEvent.request = VIM_HOST_INSTANCE_NOTIFY; + instInfo_ptr->vimEvent.operation = OPER__HOST_INST_NOTIFY ; + instInfo_ptr->vimEvent.token.url = URL_VIM_INST_LABEL ; + instInfo_ptr->vimEvent.token.url.append(instance_uuid) ; + + /* The type of HTTP request */ + instInfo_ptr->vimEvent.type = EVHTTP_REQ_PATCH ; + + /* Build the payload */ + instInfo_ptr->vimEvent.payload = ("{\"uuid\":\""); + instInfo_ptr->vimEvent.payload.append (instance_uuid); + instInfo_ptr->vimEvent.payload.append ("\",\"event-type\": \"action\",\"event-data\": {\"action\": \""); + instInfo_ptr->vimEvent.payload.append (action); + instInfo_ptr->vimEvent.payload.append ("\", \"guest-response\": \""); + instInfo_ptr->vimEvent.payload.append (guest_response); + instInfo_ptr->vimEvent.payload.append ("\", \"reason\": \""); + instInfo_ptr->vimEvent.payload.append (jsonUtil_escapeSpecialChar(reason)); + instInfo_ptr->vimEvent.payload.append ("\"}}"); + + jlog ("%s %s Payload: %s\n", hostname.c_str(), instance_uuid.c_str(), instInfo_ptr->vimEvent.payload.c_str()); + + return (guestHttpUtil_api_req ( instInfo_ptr->vimEvent )); +} + + +/***************************************************************************** + * + * Name : guestVimApi_getHostState + * + * Description: Ask the VIM for the top level fault reporting + * state for this host + * + *****************************************************************************/ + +int guestVimApi_getHostState ( string hostname, string uuid, libEvent & event ) +{ + httpUtil_event_init ( &event, + hostname, + VIM_SIG, + URL_VIM_ADDRESS, + daemon_get_cfg_ptr()->vim_event_port); + + event.base = NULL ; + event.conn = NULL ; + event.uuid = uuid ; + event.active = true ; + event.noncritical = false ; + + event.type = EVHTTP_REQ_GET ; + event.request = VIM_HOST_STATE_QUERY; + event.operation = OPER__HOST_STATE_QUERY ; + event.token.url = URL_VIM_HOST_LABEL ; + event.token.url.append(event.uuid); + + /* Build the payload */ + event.payload = "{\"hostname\": \""; + event.payload.append (hostname) ; + event.payload.append ("\",\"uuid\":\""); + event.payload.append (uuid); + event.payload.append ("\"}"); + + jlog ("%s %s Payload: %s\n", hostname.c_str(), uuid.c_str(), event.payload.c_str()); + + return ( guestHttpUtil_api_req ( event ) ); +} + + +/***************************************************************************** + * + * Name : guestVimApi_getHostInst + * + * Description: Ask the VIM for all the VM instance info for the + * specified host. + * + *****************************************************************************/ +int guestVimApi_getHostInst ( string hostname, string uuid, libEvent & event ) +{ + httpUtil_event_init ( &event, + hostname, + VIM_SIG, + URL_VIM_ADDRESS, + daemon_get_cfg_ptr()->vim_event_port); + + event.base = NULL ; + event.conn = NULL ; + event.uuid = uuid ; + event.active = true ; + event.noncritical = false ; + + event.type = EVHTTP_REQ_GET ; + event.request = VIM_HOST_INSTANCE_QUERY; + event.operation = OPER__HOST_INST_QUERY ; + event.token.url = URL_VIM_INST_LABEL ; + event.token.url.append("?host_uuid="); + event.token.url.append(event.uuid); + + jlog ("%s %s Payload: %s\n", hostname.c_str(), event.uuid.c_str(), event.token.url.c_str()); + + return ( guestHttpUtil_api_req ( event ) ); +} diff --git a/mtce-guest/src/guestVimApi.h b/mtce-guest/src/guestVimApi.h new file mode 100644 index 00000000..176fa203 --- /dev/null +++ b/mtce-guest/src/guestVimApi.h @@ -0,0 +1,47 @@ +#ifndef __INCLUDE_GUESTVIMAPI_H__ +#define __INCLUDE_GUESTVIMAPI_H__ +/* + * Copyright (c) 2013, 2015 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* + */ + +#include +#include + +#include "guestHttpUtil.h" + + /** + * @file + * Wind River CGTS Platform Guest Services Request Transmitter. + * + * This module is used by the guestAgent only and allows the guestAgent to + * + * 1. Transmit notification of an instance failure to the VIM + * + * guestVimApi_inst_failed + * + * 2. Get the instrance info for a specified host from the VIM + * + * guestVimApi_getHostState + * + * 3. Get the host level fault reporting state. + * + * guestVimApi_getHostInst + * + **************************************************************************/ + +int guestVimApi_init ( string ip, int port ); +void guestVimApi_fini ( void ); + +int guestVimApi_inst_failed ( string hostname, string instance, unsigned int event, int retries ); +int guestVimApi_inst_action ( string hostname, string instance_uuid, string action, string guest_response, string reason, int retries=0 ); +int guestVimApi_svc_event ( string hostname, string instance_uuid, string state, string status, string timeout ); +int guestVimApi_alarm_event ( string hostname, string instance_uuid ); +int guestVimApi_getHostInst ( string hostname, string uuid, libEvent & event ); +int guestVimApi_getHostState ( string hostname, string uuid, libEvent & event ); + +void guestVimApi_Handler ( struct evhttp_request *req, void *arg ); + +#endif /* __INCLUDE_GUESTVIMAPI_H__ */ diff --git a/mtce-guest/src/guestVirtio.cpp b/mtce-guest/src/guestVirtio.cpp new file mode 100644 index 00000000..88f5f4e2 --- /dev/null +++ b/mtce-guest/src/guestVirtio.cpp @@ -0,0 +1,373 @@ +/* +* Copyright (c) 2013-2015 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* +*/ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +#include "nodeBase.h" +#include "nodeEvent.h" +#include "guestBase.h" +#include "guestUtil.h" +#include "guestVirtio.h" +#include "guestInstClass.h" /* for ... get_inst */ + +/***************************************************************************** + * Name : virtio_check_filename + * + * Purpose: Return valid virtio instance heartbeat messaging socket filenames + * + * Description: + * + * Check a filename, already striped of an directory component, + * against the expected pattern for a cgcs heartbeat vio socket file. + * + * If satisfied, returns an allocated buffer containing the qemu instance name. + * The buffer must be free'd. + * + * Returns NULL on failure. + * + *****************************************************************************/ + +const char* host_virtio_dir = "/var/lib/libvirt/qemu"; + +// Use instance id to substitute the first %s below +const char* host_virtio_file_format_print = "cgcs.heartbeat.%s.sock"; +const char* alt_host_virtio_file_format_print = "wrs.heartbeat.agent.0.%s.sock"; + +// Must return '2' when scaned, first buffer recieves instance id, second should get a k, and third is unused +const char* virtio_file_format_scan = "%m[cgcs].%m[heartbeat].%m[^.].soc%m[k]%ms"; +const char* host_virtio_file_format_scan = "cgcs.heartbeat.%m[^.].soc%m[k]%ms"; +const char* alt_host_virtio_file_format_scan = "wrs.heartbeat.agent.0.%m[^.].soc%m[k]%ms"; + +string virtio_instance_name ( char * fn ) +{ + string name = "" ; + char *s1 = NULL; + char *s2= NULL; + char *instance_name = NULL; + + int rc = sscanf(fn, host_virtio_file_format_scan, &instance_name, &s1, &s2); + if (rc != 2) + { + dlog3 ("'%s' does not satisfy scan pattern %s\n", fn, host_virtio_file_format_scan); + if (s1) + { + free(s1); + s1 = NULL; + } + + if (s2) + { + free(s2); + s2 = NULL; + } + + if (instance_name) + { + free(instance_name); + instance_name = NULL; + } + + rc = sscanf(fn, alt_host_virtio_file_format_scan, &instance_name, &s1, &s2); + if (rc != 2) + { + dlog3 ("'%s' does not satisfy scan pattern %s\n", fn, alt_host_virtio_file_format_scan); + if (instance_name) + { + free(instance_name); + instance_name = NULL; + } + } + else + { + /* Valid instance filename found */ + name = instance_name ; + } + } + else + { + /* Valid instance filename found */ + name = instance_name ; + } + + if (s1) free(s1); + if (s2) free(s2); + + if (instance_name) + { + free(instance_name); + } + + return (name); +} + + +bool virtio_check_filename ( char * fn ) +{ + string instance_name = virtio_instance_name ( fn ) ; + if ( instance_name.size () == UUID_LEN ) + return true ; + else + return false ; +} + +/* Add the auto detected channel to the instance list + * WARNING: This is where the cgcs.heartbeat.*.sock part is + * removed from the channel and put into the instInfo + * struct as a uuid value */ +int virtio_channel_add ( char * channel ) +{ + instInfo * instInfo_ptr ; + int rc = FAIL_NOT_FOUND ; + char * prefix1 = NULL ; + char * prefix2 = NULL ; + char * suffix = NULL ; + char * uuid_ptr = NULL ; + char * s1 = NULL ; + string uuid = ""; + instInfo instance ; + guestUtil_inst_init ( &instance ); + + rc = sscanf(channel, virtio_file_format_scan, &prefix1, &prefix2, &uuid_ptr, &suffix, &s1 ); + if ( rc != 4 ) + { + elog ("failed to extract uuid from channel %s (num:%d)\n", channel, rc); + rc = FAIL_INVALID_DATA ; + goto virtio_channel_add_cleanup ; + } + + uuid = uuid_ptr ; + if ( uuid.length() != UUID_LEN ) + { + elog ("failed to get UUID from channel %s (uuid:%ld)\n", uuid.c_str(), uuid.length()); + rc = FAIL_INVALID_UUID ; + goto virtio_channel_add_cleanup ; + } + + + instInfo_ptr = get_instInv_ptr()->get_inst ( uuid ); + if ( instInfo_ptr ) + { + /* detected channel found */ + ilog ("%s add ; already provisioned\n", log_prefix(instInfo_ptr).c_str()); + rc = PASS ; + } + else if ( ( rc = get_instInv_ptr()->add_inst ( uuid, instance ) ) == PASS ) + { + dlog ("%s add ; auto provisioned\n", instance.uuid.c_str()); + rc = PASS ; + } + else + { + elog ("%s add failed\n", uuid.c_str()); + rc = FAIL_INVALID_UUID ; + } + + if ( rc == PASS ) + { + + /* get the recently added instance */ + instInfo_ptr = get_instInv_ptr()->get_inst ( uuid ); + if ( instInfo_ptr ) + { + instInfo_ptr->uuid = uuid ; + instInfo_ptr->chan = channel ; + instInfo_ptr->fd_namespace = QEMU_CHANNEL_DIR ; + instInfo_ptr->fd_namespace.append ("/") ; + instInfo_ptr->fd_namespace.append (channel) ; + + instInfo_ptr->connect_wait_in_secs = DEFAULT_CONNECT_WAIT ; + + get_instInv_ptr()->reconnect_start ( (const char *)uuid_ptr ) ; + } + } + +virtio_channel_add_cleanup: + + if (prefix1) free(prefix1); + if (prefix2) free(prefix2); + if (suffix) free(suffix); + if (uuid_ptr) free(uuid_ptr); + if (s1) free (s1); + + return(rc); +} + + + +/***************************************************************************** + * + * Name : virtio_channel_connect + * + * Purpose : Connect to the channel specified by the instance pointer + * + *****************************************************************************/ +int virtio_channel_connect ( instInfo * instInfo_ptr ) +{ + int rc = PASS ; + char buf[PATH_MAX]; + + if ( ! instInfo_ptr ) + { + slog ("called with NULL instance pointer\n"); + return (FAIL_NULL_POINTER); + } + + snprintf(buf, sizeof(buf), "%s/cgcs.heartbeat.%s.sock", QEMU_CHANNEL_DIR, instInfo_ptr->uuid.data()); + + dlog ("... trying connect: %s\n", buf ); + + if (( instInfo_ptr->chan_fd > 0 ) && ( instInfo_ptr->chan_ok == true )) + { + if ( instInfo_ptr->connected ) + { + ilog ("%s already connected\n", log_prefix(instInfo_ptr).c_str()); + return (PASS); + } + else + { + ilog ("%s socket and chan ok but not connected\n", log_prefix(instInfo_ptr).c_str()); + } + } + + instInfo_ptr->chan_ok = false ; + instInfo_ptr->connected = false ; + + if ( instInfo_ptr->chan_fd ) + close (instInfo_ptr->chan_fd); + + /* found channel */ + instInfo_ptr->chan_fd = socket ( AF_UNIX, CHAN_FLAGS, 0 ); + if ( instInfo_ptr->chan_fd <= 0 ) + { + ilog("%s socket create failed for %s, (%d:%m)\n", log_prefix(instInfo_ptr).c_str(), buf, errno ) ; + rc = FAIL_SOCKET_CREATE ; + } + else + { + int flags ; + struct linger so_linger ; + + /* get socket flags */ + flags = fcntl(instInfo_ptr->chan_fd, F_GETFL); + if (flags < 0) + { + elog ("%s failed to get socket %d flags (%d:%m)\n", + log_prefix(instInfo_ptr).c_str(), + instInfo_ptr->chan_fd , errno); + rc = FAIL_SOCKET_OPTION ; + } + + /* set socket as nonblocking */ + if ( flags & O_NONBLOCK ) + { + dlog ("%s Socket already set as non-blocking\n", + log_prefix(instInfo_ptr).c_str()); + } + else + { + flags = (flags | O_NONBLOCK); + if (fcntl(instInfo_ptr->chan_fd, F_SETFL, flags) < 0) + { + elog ("%s failed to set socket %d nonblocking (%d:%m)\n", + instInfo_ptr->uuid.data(), + instInfo_ptr->chan_fd , errno); + rc = FAIL_SOCKET_NOBLOCK ; + } + } + so_linger.l_onoff = 1 ; /* true */ + so_linger.l_linger = 0 ; /* linger time is 0 ; no TIME_WAIT */ + + rc = setsockopt ( instInfo_ptr->chan_fd, SOL_SOCKET, SO_LINGER, &so_linger, sizeof(so_linger)); + if ( rc ) + { + elog ("%s failed to set linger=0 option (%d:%m)\n", log_prefix(instInfo_ptr).c_str(), errno ); + } + } + + if ( rc == PASS ) + { + int len ; + struct sockaddr_un un; + un.sun_family = AF_UNIX; + + strcpy(un.sun_path, buf); + len = offsetof(struct sockaddr_un, sun_path) + strlen(buf); + rc = connect(instInfo_ptr->chan_fd, (struct sockaddr *)&un, len); + if (rc < 0) + { + elog ( "%s connect failed %s (%d:%d:%m)\n", + log_prefix(instInfo_ptr).c_str(), buf, rc, errno); + } + else + { + ilog ("%s connect accepted\n", log_prefix(instInfo_ptr).c_str() ); + instInfo_ptr->chan_ok = true ; + instInfo_ptr->connected = true ; + rc = PASS ; + } + } + /* Handle errors */ + if ( rc != PASS ) + { + /* TODO: cleanup */ + if (instInfo_ptr->chan_fd ) + { + ilog ("%s closing socket %d\n", + log_prefix(instInfo_ptr).c_str(), + instInfo_ptr->chan_fd); + + close (instInfo_ptr->chan_fd) ; + instInfo_ptr->chan_fd = 0 ; + instInfo_ptr->chan_ok = false ; + instInfo_ptr->connected = false ; + } + /* TODO: consider removing this entry from the list */ + } + return (rc); +} + + +int virtio_channel_connect ( string channel ) +{ + instInfo * instInfo_ptr = get_instInv_ptr()->get_inst ( channel ) ; + if ( instInfo_ptr ) + { + return ( virtio_channel_connect ( instInfo_ptr )); + } + elog ("%s instance lookup failed\n", channel.c_str() ); + return (FAIL_NULL_POINTER); +} diff --git a/mtce-guest/src/guestVirtio.h b/mtce-guest/src/guestVirtio.h new file mode 100644 index 00000000..82f31382 --- /dev/null +++ b/mtce-guest/src/guestVirtio.h @@ -0,0 +1,24 @@ +#ifndef __GUESTVIRTIO_H__ +#define __GUESTVIRTIO_H__ + +/* +* Copyright (c) 2013-2015 Wind River Systems, Inc. +* +* SPDX-License-Identifier: Apache-2.0 +* +*/ + +#include +#include + +using namespace std; + +#include "guestBase.h" + +bool virtio_check_filename ( char * fn ); +int virtio_channel_connect ( string channel ); +int virtio_channel_connect ( instInfo * inst_ptr ); +int virtio_channel_add ( char * chan_ptr ); +string virtio_instance_name ( char * fn ); + +#endif /* __GUESTVIRTIO_H__ */ diff --git a/mtce-guest/src/scripts/guest.ini b/mtce-guest/src/scripts/guest.ini new file mode 100644 index 00000000..26e666fd --- /dev/null +++ b/mtce-guest/src/scripts/guest.ini @@ -0,0 +1,28 @@ +; CGTS Guest Service daemons config file +[agent] ; Agent Configuration +rx_port = 2401 ; guestAgent inter-daemon messaging rx port number +vim_cmd_port = 2410 ; vim to guestAgent command port +hbs_failure_threshold = 1 ; Number of failures to accept before reporting the fault + +[client] ; Client Configuration +rx_port = 2411 ; guestAgent inter-daemon messaging rx port number +hbs_failure_threshold = 1 ; Number of failures to accept before reporting the fault +hbs_pulse_period = 400 ; Smallest allowable heartbeat interval in msecs + +[timeouts] ; Configurable timeouts - values in seconds +start_delay = 1 ; time in secs to wait before starting failure reporting + +[debug] ; +debug_timer = 0 ; enable(1) or disable(0) timer logs (tlog) +debug_json = 0 ; enable(1) or disable(0) Json logs (jlog) +debug_fsm = 0 ; enable(1) or disable(0) fsm logs (flog) +debug_http = 0 ; enable(1) or disable(0) http logs (hlog) +debug_alive = 0 ; enable(1) or disable(0) mtcAlive logs (alog) +debug_msg = 0 ; enable(1) or disable(0) message logs (mlog) +debug_state = 0 ; enable(1) or disable(0) state change logs (clog) +debug_work = 0 ; enable(1) or disable(0) work queue trace logs (qlog) +debug_level = 0 ; decimal mask 0..15 (8,4,2,1) and 16 for mem logging +debug_all = 0 ; set all debug labels to the specified value + +flush = 1 ; enable(1) or disable(0) force log flush (main loop) +flush_thld = 5 ; if enabled - force flush after this number of loops diff --git a/mtce-guest/src/scripts/guestAgent b/mtce-guest/src/scripts/guestAgent new file mode 100644 index 00000000..364e1e71 --- /dev/null +++ b/mtce-guest/src/scripts/guestAgent @@ -0,0 +1,113 @@ +#! /bin/sh +# +# Copyright (c) 2013-2014, 2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# chkconfig: 2345 95 95 +# +### BEGIN INIT INFO +# Provides: guestAgent +# Default-Start: 3 5 +# Default-Stop: 0 1 2 6 +# Short-Description: Maintenance Client Daemon +### END INIT INFO + +. /etc/init.d/functions + +DAEMON_NAME="guestAgent" +DAEMON="/usr/local/bin/${DAEMON_NAME}" +PIDFILE="/var/run/${DAEMON_NAME}.pid" +PLATFORM_CONF="/etc/platform/platform.conf" + +IFACE="" + +# Linux Standard Base (LSB) Error Codes +RETVAL=0 +GENERIC_ERROR=1 +INVALID_ARGS=2 +UNSUPPORTED_FEATURE=3 +NOT_INSTALLED=5 +NOT_RUNNING=7 + +PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin +export PATH + +if [ ! -e "${DAEMON}" ] ; then + logger "${DAEMON} is missing" + exit ${NOT_INSTALLED} +fi + +if [ -f ${PLATFORM_CONF} ] ; then + IFACE=`cat ${PLATFORM_CONF} | grep management_interface | cut -f2 -d'='` + if [ "${IFACE}" != "" ] ; then + if ip link show $IFACE | grep -sq 'state DOWN'; then + ip link set dev $IFACE up + fi + fi +fi + +case "$1" in + start) + logger "Starting ${DAEMON_NAME}" + echo -n "Starting ${DAEMON_NAME}: " + if [ -n "`pidof ${DAEMON_NAME}`" ] ; then + echo -n "is already running " + RETVAL=0 + else + start-stop-daemon --start -b -x ${DAEMON} -- -l + RETVAL=$? + fi + if [ ${RETVAL} -eq 0 ] ; then + pid=`pidof ${DAEMON_NAME}` + echo "OK" + logger "${DAEMON} (${pid})" + else + echo "FAIL" + RETVAL=${GENERIC_ERROR} + fi + ;; + + stop) + logger "Stopping ${DAEMON_NAME}" + echo -n "Stopping ${DAEMON_NAME}: " + if [ -n "`pidof ${DAEMON_NAME}`" ] ; then + killproc ${DAEMON_NAME} + fi + if [ -n "`pidof ${DAEMON_NAME}`" ] ; then + echo "FAIL" + RETVAL=${NOT_RUNNING} + else + echo "OK" + fi + rm -f ${PIDFILE} + ;; + + restart) + $0 stop + $0 start + ;; + + status) + pid=`pidof ${DAEMON_NAME}` + RETVAL=$? + if [ ${RETVAL} -eq 0 ] ; then + echo "${DAEMON_NAME} is running" + else + echo "${DAEMON_NAME} is NOT running" + RETVAL=${NOT_RUNNING} + fi + ;; + + condrestart) + $0 restart + ;; + + *) + echo "usage: $0 { start | stop | status | restart | condrestart | status }" + ;; +esac + +exit ${RETVAL} diff --git a/mtce-guest/src/scripts/guestAgent.logrotate b/mtce-guest/src/scripts/guestAgent.logrotate new file mode 100644 index 00000000..654a7135 --- /dev/null +++ b/mtce-guest/src/scripts/guestAgent.logrotate @@ -0,0 +1,16 @@ +#daily + +/var/log/guestAgent.log +{ + nodateext + size 10M + rotate 5 + start 1 + missingok + notifempty + compress + sharedscripts + postrotate + systemctl reload syslog-ng > /dev/null 2>&1 || true + endscript +} diff --git a/mtce-guest/src/scripts/guestAgent.ocf b/mtce-guest/src/scripts/guestAgent.ocf new file mode 100644 index 00000000..34708a87 --- /dev/null +++ b/mtce-guest/src/scripts/guestAgent.ocf @@ -0,0 +1,442 @@ +#!/bin/sh +# +# Copyright (c) 2013-2017 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# Support: www.windriver.com +# +# Purpose: This resource agent manages +# +# .... the Titanium Cloud Controller Maintenance Daemon +# +# RA Spec: +# +# http://www.opencf.org/cgi-bin/viewcvs.cgi/specs/ra/resource-agent-api.txt?rev=HEAD +# +####################################################################### +# Initialization: + +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs + + +####################################################################### + +# Fill in some defaults if no values are specified +OCF_RESKEY_binary_default="guestAgent" +OCF_RESKEY_config_default="/etc/mtc/guestAgent.ini" +OCF_RESKEY_dbg_default="false" +OCF_RESKEY_mode_default="normal" +OCF_RESKEY_user_default="admin" +OCF_RESKEY_pid_default="/var/run/guestAgent.pid" + + +: ${OCF_RESKEY_binary=${OCF_RESKEY_binary_default}} +: ${OCF_RESKEY_config=${OCF_RESKEY_config_default}} +: ${OCF_RESKEY_dbg=${OCF_RESKEY_dbg_default}} +: ${OCF_RESKEY_mode=${OCF_RESKEY_mode_default}} +: ${OCF_RESKEY_user=${OCF_RESKEY_user_default}} +: ${OCF_RESKEY_pid=${OCF_RESKEY_pid_default}} + +mydaemon="/usr/local/bin/${OCF_RESKEY_binary}" +statusfile="/var/run/${OCF_RESKEY_binary}.info" + +####################################################################### + +usage() { + cat < + + +1.0 + + +This 'guestAgent' is an OCF Compliant Resource Agent that manages start, stop and in- +service monitoring of Maintenance's guestAgent daemon on Wind River's Titanium Cloud. + + + +Manages the Titanium Cloud's Maintenance guestAgent service daemon. + + + + + + + +mode = normal ... run maintenance daemon in 'normal' mode (default) +mode = passive ... run maintenance daemon in 'passive' mode + +Maintenance Mode Option + + + + + + +dbg = false ... info, warn and err logs sent to output stream (default) +dbg = true ... Additional dbg logs are also sent to the output stream + +Service Debug Control Option + + + + + + + + + + + + + + +END + return ${OCF_SUCCESS} +} + +guestAgent_validate() { + + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "guestAgent:validate" + fi + + check_binary "/usr/local/bin/${OCF_RESKEY_binary}" + check_binary pidof + + if [ ! -f ${OCF_RESKEY_config} ] ; then + msg="${OCF_RESKEY_binary} file missing ${OCF_RESKEY_config}" + ocf_log err "${msg}" + return ${OCF_ERR_CONFIGURED} + fi + + return ${OCF_SUCCESS} +} + +guestAgent_status () { + + proc="guestAgent:status" + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "guestAgent:status" + fi + + # remove the status file before we request a new + rm -f ${statusfile} + + # Verify the pid file exists as part of status + for ((loop=0;loop<3;loop++)) { + if [ -f ${OCF_RESKEY_pid} ] ; then + break + else + sleep 1 + fi + } + + # See if the daemon is running + pid=`cat ${OCF_RESKEY_pid}` + kill -0 $pid 2> /dev/null + if [ $? -eq 0 ] ; then + + log_sig="${OCF_RESKEY_binary} In-Service Active Monitor Test" + + # Ask the daemon to produce status + ocf_run kill -s USR1 $pid + + # Wait for the response + for ((loop=0;loop<10;loop++)) { + sleep 1 + if [ -f ${statusfile} ] ; then + + ocf_log info "${log_sig} Passed ($loop)" + return ${OCF_SUCCESS} + + elif [ $loop -eq 5 ] ; then + + # send the signal again + ocf_run kill -s USR1 $pid + + pid_stat=`cat /proc/${pid}/stat` + ocf_log notice "${log_sig} is slow to respond" + ocf_log notice "$pid_stat" + + elif [ $loop -eq 8 ] ; then + + pid_stat=`cat /proc/${pid}/stat` + ocf_log warn "${log_sig} is very slow to respond" + ocf_log warn "$pid_stat" + + fi + } + log_procfs + ocf_log err "${log_sig} Failed" + return ${OCF_ERR_GENERIC} + fi + return ${OCF_NOT_RUNNING} +} + +guestAgent_monitor () { + + proc="guestAgent:monitor" + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${proc}" + fi + + # Uncomment if you want the monitor function to force-pass + # return ${OCF_SUCCESS} + + pid=`cat ${OCF_RESKEY_pid}` + kill -0 $pid 2> /dev/null + if [ $? -ne 0 ] ; then + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${proc} called while ${OCF_RESKEY_binary} not running." + fi + return ${OCF_NOT_RUNNING} + fi + + guestAgent_status + return $? +} + + +guestAgent_start () { + + local rc + + start_proc="guestAgent:start" + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${start_proc}" + fi + + # Uncomment if you want the start function to force-pass without starting + # return ${OCF_SUCCESS} + + # If running then issue a ping test + pid=`cat ${OCF_RESKEY_pid}` + kill -0 $pid 2> /dev/null + if [ $? -eq 0 ] ; then + guestAgent_status + rc=$? + if [ $rc -ne ${OCF_SUCCESS} ] ; then + msg="${start_proc} ping test failed rc=${rc}" + ocf_log err "${msg}" + guestAgent_stop + else + # Spec says to return success if process is already running for start + pid=`cat ${OCF_RESKEY_pid}` + kill -0 $pid 2> /dev/null + if [ $? -eq 0 ] ; then + ocf_log info "${start_proc} called while ${OCF_RESKEY_binary} is already running" + return ${OCF_SUCCESS} + fi + fi + fi + + # should not be running now or error + pid=`cat ${OCF_RESKEY_pid}` + kill -0 $pid 2> /dev/null + if [ $? -eq 0 ] ; then + msg="${start_proc} cannot kill off existing instance of ${OCF_RESKEY_binary}" + ocf_log err "${msg}" + return ${OCF_RUNNING_MASTER} + fi + + rm -f ${statusfile} + + # default PID to null + pid="" + + # Try to Start the daemon + ${mydaemon} + rc=$? + + # verify it was started and set return code appropriately + if [ $rc -eq ${OCF_SUCCESS} ] ; then + # Verify the pid file exists as part of status + for ((loop=0;loop<3;loop++)) { + if [ -f ${OCF_RESKEY_pid} ] ; then + break + else + ocf_log info "${start_proc} waiting ... loop=${loop}" + sleep 1 + fi + } + + pid=`cat ${OCF_RESKEY_pid}` + # ocf_log info "PID:$pid" + kill -0 $pid 2> /dev/null + if [ $? -ne 0 ] ; then + rc=${OCF_FAILED_MASTER} + else + if [ ! -f ${statusfile} ] ; then + ocf_log info "guestAgent: Startup Health Test Failed - missing info" + rc = ${OCF_ERR_GENERIC} + fi + fi + else + ocf_log info "${start_proc} failed ${mydaemon} daemon rc=${rc}" + rc = ${OCF_ERR_GENERIC} + fi + + # Record success or failure and return status + if [ ${rc} -eq $OCF_SUCCESS ] ; then + msg="${start_proc}ed pid=${pid}" + ocf_log info "${msg}" + else + msg="${start_proc} failed rc=${rc}" + ocf_log err "${msg}" + rc=${OCF_NOT_RUNNING} + fi + return $rc +} + +guestAgent_confirm_stop () { + + proc="guestAgent:confirm_stop" + ocf_log info "${proc}" + + pid=`pidof ${OCF_RESKEY_binary}` + kill -0 ${pid} 2> /dev/null + if [ $? -eq 0 ] ; then + ocf_log info "${proc} 'kill -9 ${pid}'" + kill -9 ${pid} + ocf_log info "${proc}ed (by emergency kill -9 ${pid})" + sleep 1 + fi + rm -f ${OCF_RESKEY_pid} +} + +guestAgent_stop () { + + proc="guestAgent:stop" + + # See if the process is running by pidfile + + pid=`pidof ${OCF_RESKEY_binary}` + ocf_log info "${proc} PID:${pid}" + kill -0 ${pid} 2> /dev/null + if [ $? -ne 0 ] ; then + ocf_log info "${proc} called while already stopped (no process)" + guestAgent_confirm_stop + return ${OCF_SUCCESS} + fi + + MAX=3 + for ((loop=0;loop<$MAX;loop++)) { + + # verify stop with pidfile + if [ -f ${OCF_RESKEY_pid} ] ; then + + pid=`cat ${OCF_RESKEY_pid}` + + # if pid file is gone we are done + if [ ${pid} = "" ] ; then + ocf_log info "${proc}ped (by -int)" + break + + # if pidfile is empty then kill by -int + else + + kill -0 ${pid} 2> /dev/null + if [ $? -ne 0 ] ; then + ocf_log info "${proc}ped (by pid)" + break + else + ocf_log info "${proc}ping (by -int - loop:${loop})" + kill -int ${pid} + sleep 1 + fi + fi + fi + } + guestAgent_confirm_stop + return ${OCF_SUCCESS} +} + +guestAgent_reload () { + + local rc + + proc="guestAgent:reload" + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${proc}" + fi + + guestAgent_stop + rc=$? + if [ $rc -eq ${OCF_SUCCESS} ] ; then + #sleep 1 + guestAgent_start + rc=$? + if [ $rc -eq ${OCF_SUCCESS} ] ; then + msg="${proc}ed" + ocf_log info "${mgs}" + fi + fi + + if [ ${rc} -ne ${OCF_SUCCESS} ] ; then + msg="${OCF_RESKEY_binary}: failed to restart rc=${rc}" + ocf_log info "${mgs}" + fi + + return ${rc} +} + +case ${__OCF_ACTION} in + meta-data) meta_data + exit ${OCF_SUCCESS} + ;; + usage|help) usage + exit ${OCF_SUCCESS} + ;; +esac + +ocf_log info "guestAgent:${__OCF_ACTION} action" + +# Anything except meta-data and help must pass validation +guestAgent_validate || exit $? + +case ${__OCF_ACTION} in + start) guestAgent_start + ;; + stop) guestAgent_stop + ;; + status) guestAgent_status + ;; + reload) guestAgent_reload + ;; + monitor) guestAgent_monitor + ;; + validate-all) guestAgent_validate + ;; + *) usage + exit ${OCF_ERR_UNIMPLEMENTED} + ;; +esac diff --git a/mtce-guest/src/scripts/guestAgent.service b/mtce-guest/src/scripts/guestAgent.service new file mode 100644 index 00000000..e88a965b --- /dev/null +++ b/mtce-guest/src/scripts/guestAgent.service @@ -0,0 +1,16 @@ +[Unit] +Description=Titanium Cloud Guest Agent +After=network.target syslog.service + +[Service] +Type=forking +ExecStart=/etc/rc.d/init.d/guestAgent start +ExecStop=/etc/rc.d/init.d/guestAgent stop +ExecReload=/etc/rc.d/init.d/guestAgent reload +PIDFile=/var/run/guestAgent.pid + +Restart=no + +[Install] +WantedBy=multi-user.target + diff --git a/mtce-guest/src/scripts/guestAgentTest.sh b/mtce-guest/src/scripts/guestAgentTest.sh new file mode 100644 index 00000000..a4994e0f --- /dev/null +++ b/mtce-guest/src/scripts/guestAgentTest.sh @@ -0,0 +1,160 @@ +#! /bin/bash + +# +# Copyright (c) 2015 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +echo "Running guestAgent guest services command testhead" + +if [ -z $1 ] ; then + echo "Error: must supply a host name as first arguement" + echo "Syntax: $0 compute-1" + exit 1 +fi + +echo "Args: $1 $2 $3" + +banner="-----------------------------------------------------------" + +hostname=$1 +hostuuid=`system host-show $hostname | grep uuid | cut -f 15 -d ' '` +#hostuuid=`system host-show $hostname | grep uuid` + +echo "hostname: $hostname" +echo "hostuuid: $hostuuid" +echo "Emulating VIM guest services commands against $hostname" + +count=1 + +echo $banner +echo "$count Create Host Services" +echo $banner +curl -i -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/hosts/$hostuuid + +count=$((count + 1)) + +echo $banner +echo "$count Query Host Services" +echo $banner +curl -i -X GET -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/hosts/$hostuuid + +count=$((count + 1)) + +echo $banner +echo "$count Enable Host Services" +echo $banner +curl -i -X PUT -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/hosts/$hostuuid/enable -d '{"hostname": "compute-1", "uuid" : "010e7741-1173-4a3b-88fa-c4e5905500ca"}' + +count=$((count + 1)) + +echo $banner +echo "$count Create Guest Service: Instance 1" +echo $banner +curl -i -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/instances/8d80875b-fa73-4ccb-bce3-1cd4df104400 -d '{"hostname": "compute-1", "uuid" : "8d80875b-fa73-4ccb-bce3-1cd4df104400", "channel" : "cgts-instance000001", "services" : ["heartbeat"]}' + +count=$((count + 1)) + +echo $banner +echo "$count Create Guest Service: Instance 2" +echo $banner +curl -i -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/instances/8d80875b-fa73-4ccb-bce3-1cd4df104401 -d '{"hostname": "compute-1", "uuid" : "8d80875b-fa73-4ccb-bce3-1cd4df104401", "channel" : "cgts-instance000002", "services" : ["heartbeat"]}' + +count=$((count + 1)) + +echo $banner +echo "$count Query Guest Services: Instance 2:" +echo $banner +curl -i -X GET -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/instances/8d80875b-fa73-4ccb-bce3-1cd4df104401 + +count=$((count + 1)) + +echo $banner +echo "$count Query Guest Services: Instance 1:" +echo $banner +curl -i -X GET -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/instances/8d80875b-fa73-4ccb-bce3-1cd4df104400 + +count=$((count + 1)) + +echo $banner +echo "$count Enable Guest Service: Instance 2" +echo $banner +curl -i -X PATCH -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/instances/8d80875b-fa73-4ccb-bce3-1cd4df104401 -d '{"hostname": "compute-1", "uuid" : "8d80875b-fa73-4ccb-bce3-1cd4df104401", "channel" : "cgts-instance000002", "services" : [{"service":"heartbeat" , "state":"enabled"}]}' + +count=$((count + 1)) + +echo $banner +echo "$count Query Guest Services: Instance 2:" +echo $banner +curl -i -X GET -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/instances/8d80875b-fa73-4ccb-bce3-1cd4df104401 + +count=$((count + 1)) + +echo $banner +echo "$count Disable Guest Service: Instance 2" +echo $banner +curl -i -X PATCH -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/instances/8d80875b-fa73-4ccb-bce3-1cd4df104401 -d '{"hostname": "compute-1", "uuid" : "8d80875b-fa73-4ccb-bce3-1cd4df104401", "channel" : "cgts-instance000002", "services" : [{"service":"heartbeat" , "state":"disabled"}]}' + +count=$((count + 1)) + +echo $banner +echo "$count Query Guest Services: Instance 1:" +echo $banner +curl -i -X GET -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/instances/8d80875b-fa73-4ccb-bce3-1cd4df104401 + +count=$((count + 1)) + +exit 0 + +echo $banner +echo "$count Delete Guest Service: Instance 2" +echo $banner +curl -i -X DELETE -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/instances/8d80875b-fa73-4ccb-bce3-1cd4df104401 + +count=$((count + 1)) + +echo $banner +echo "$count Query Host Services" +echo $banner +curl -i -X GET -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/hosts/$hostuuid + +count=$((count + 1)) + +echo $banner +echo "$count Disable Host Services" +echo $banner +curl -i -X PUT -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/hosts/$hostuuid/disable -d '{"hostname": "compute-1", "uuid" : "010e7741-1173-4a3b-88fa-c4e5905500ca"}' + +count=$((count + 1)) + +echo $banner +echo "$count Delete Host Services" +echo $banner +curl -i -X DELETE -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/hosts/$hostuuid + +count=$((count + 1)) + +echo $banner +echo "$count Enable Guest Service: Instance 1" +echo $banner +curl -i -X PATCH -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/instances/8d80875b-fa73-4ccb-bce3-1cd4df104400 -d '{"hostname": "compute-1", "uuid" : "8d80875b-fa73-4ccb-bce3-1cd4df104400", "channel" : "cgts-instance000001", "services" : [{"service":"heartbeat" , "state":"enabled"}]}' + +count=$((count + 1)) + +echo $banner +echo "$count Disable Guest Service: Instance 1" +echo $banner +curl -i -X PATCH -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/instances/8d80875b-fa73-4ccb-bce3-1cd4df104400 -d '{"hostname": "compute-1", "uuid" : "8d80875b-fa73-4ccb-bce3-1cd4df104400", "channel" : "cgts-instance000001", "services" : [{"service":"heartbeat" , "state":"disabled"}]}' + +count=$((count + 1)) + +echo $banner +echo "$count Enable Guest Service: Instance 1 - Change Channel" +echo $banner +curl -i -X PATCH -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'User-Agent: vim/1.0' http://localhost:2410/v1/instances/8d80875b-fa73-4ccb-bce3-1cd4df104400 -d '{"hostname": "compute-1", "uuid" : "8d80875b-fa73-4ccb-bce3-1cd4df104400", "channel" : "cgts-instance000003", "services" : [{"service":"heartbeat" , "state":"enabled"}]}' + +echo $banner +echo $banner + +exit 0 diff --git a/mtce-guest/src/scripts/guestServer b/mtce-guest/src/scripts/guestServer new file mode 100644 index 00000000..5e8d0877 --- /dev/null +++ b/mtce-guest/src/scripts/guestServer @@ -0,0 +1,113 @@ +#! /bin/sh +# +# Copyright (c) 2013-2014, 2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# chkconfig: 2345 95 95 +# +### BEGIN INIT INFO +# Provides: guestServer +# Default-Start: 3 5 +# Default-Stop: 0 1 2 6 +# Short-Description: Maintenance Client Daemon +### END INIT INFO + +. /etc/init.d/functions + +DAEMON_NAME="guestServer" +DAEMON="/usr/local/bin/${DAEMON_NAME}" +PIDFILE="/var/run/${DAEMON_NAME}.pid" +PLATFORM_CONF="/etc/platform/platform.conf" + +IFACE="" + +# Linux Standard Base (LSB) Error Codes +RETVAL=0 +GENERIC_ERROR=1 +INVALID_ARGS=2 +UNSUPPORTED_FEATURE=3 +NOT_INSTALLED=5 +NOT_RUNNING=7 + +PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin +export PATH + +if [ ! -e "${DAEMON}" ] ; then + logger "${DAEMON} is missing" + exit ${NOT_INSTALLED} +fi + +if [ -f ${PLATFORM_CONF} ] ; then + IFACE=`cat ${PLATFORM_CONF} | grep management_interface | cut -f2 -d'='` + if [ "${IFACE}" != "" ] ; then + if ip link show $IFACE | grep -sq 'state DOWN'; then + ip link set dev $IFACE up + fi + fi +fi + +case "$1" in + start) + logger "Starting ${DAEMON_NAME}" + echo -n "Starting ${DAEMON_NAME}: " + if [ -n "`pidof ${DAEMON_NAME}`" ] ; then + echo -n "is already running " + RETVAL=0 + else + start-stop-daemon --start -b -x ${DAEMON} -- -l + RETVAL=$? + fi + if [ ${RETVAL} -eq 0 ] ; then + pid=`pidof ${DAEMON_NAME}` + echo "OK" + logger "${DAEMON} (${pid})" + else + echo "FAIL" + RETVAL=${GENERIC_ERROR} + fi + ;; + + stop) + logger "Stopping ${DAEMON_NAME}" + echo -n "Stopping ${DAEMON_NAME}: " + if [ -n "`pidof ${DAEMON_NAME}`" ] ; then + killproc ${DAEMON_NAME} + fi + if [ -n "`pidof ${DAEMON_NAME}`" ] ; then + echo "FAIL" + RETVAL=${NOT_RUNNING} + else + echo "OK" + fi + rm -f ${PIDFILE} + ;; + + restart) + $0 stop + $0 start + ;; + + status) + pid=`pidof ${DAEMON_NAME}` + RETVAL=$? + if [ ${RETVAL} -eq 0 ] ; then + echo "${DAEMON_NAME} is running" + else + echo "${DAEMON_NAME} is NOT running" + RETVAL=${NOT_RUNNING} + fi + ;; + + condrestart) + $0 restart + ;; + + *) + echo "usage: $0 { start | stop | status | restart | condrestart | status }" + ;; +esac + +exit ${RETVAL} diff --git a/mtce-guest/src/scripts/guestServer.logrotate b/mtce-guest/src/scripts/guestServer.logrotate new file mode 100644 index 00000000..9b54a875 --- /dev/null +++ b/mtce-guest/src/scripts/guestServer.logrotate @@ -0,0 +1,17 @@ +#daily +nodateext + +/var/log/guestServer.log +{ + nodateext + size 10M + rotate 5 + start 1 + missingok + notifempty + compress + sharedscripts + postrotate + systemctl reload syslog-ng > /dev/null 2>&1 || true + endscript +} diff --git a/mtce-guest/src/scripts/guestServer.pmon b/mtce-guest/src/scripts/guestServer.pmon new file mode 100644 index 00000000..a19ce303 --- /dev/null +++ b/mtce-guest/src/scripts/guestServer.pmon @@ -0,0 +1,25 @@ +[process] +process = guestServer +service = guestServer +pidfile = /var/run/guestServer.pid +script = /etc/init.d/guestServer +style = lsb ; ocf or lsb +severity = major ; minor, major, critical +restarts = 3 ; restart retries before error assertion +interval = 3 ; number of seconds to wait between restarts +debounce = 10 ; number of seconds that a process needs to remain + ; running before degrade is removed and retry count + ; is cleared. +startuptime = 1 ; Seconds to wait after process start before starting the debounce monitor +mode = passive ; Monitoring mode: passive (default) or active + ; passive: process death monitoring (default: always) + ; active : heartbeat monitoring, i.e. request / response messaging + ; ignore : do not monitor or stop monitoring +subfunction = compute ; Optional label. + ; Manage this process in the context of a combo host subfunction + ; Choices: compute or storage. + ; when specified pmond will wait for + ; /var/run/.compute_config_complete or + ; /var/run/.storage_config_complete + ; ... before managing this process with the specified subfunction + ; Excluding this label will cause this process to be managed by default on startup diff --git a/mtce-guest/src/scripts/guestServer.service b/mtce-guest/src/scripts/guestServer.service new file mode 100644 index 00000000..6e2dd3a2 --- /dev/null +++ b/mtce-guest/src/scripts/guestServer.service @@ -0,0 +1,23 @@ +[Unit] +Description=Titanium Cloud Maintenance Guest Heartbeat Monitor Server +After=network.target syslog.service config.service +Before=pmon.service + +[Service] +Type=forking +ExecStart=/etc/rc.d/init.d/guestServer start +ExecStop=/etc/rc.d/init.d/guestServer stop +ExecReload=/etc/rc.d/init.d/guestServer reload +PIDFile=/var/run/guestServer.pid + +# Failure handling +TimeoutStartSec=10s +TimeoutStopSec=10s + +# process recovery is handled by pmond +Restart=no +RestartSec=5 + +[Install] +WantedBy=multi-user.target +