synchronize guest logs to controller node

use rsync to synchronize guest logs to the controller node so that we can
find the logs even after the instance is deleted

Add a guest-log-collection timer, this timer triggers
guest-log-collection service every 10 seconds

Change-Id: I264235ce7b7df585358654c46bdac4dcd743fb3d

devstack/plugin.sh

@@ -463,7 +463,7 @@ function create_guest_image {
         curl -sSL ${TROVE_NON_DEV_IMAGE_URL} -o ${image_file}
     else
         echo "Starting to create guest image"
-
+        export SYNC_LOG_TO_CONTROLLER=${SYNC_LOG_TO_CONTROLLER:-"False"}
         $DEST/trove/integration/scripts/trovestack \
           build-image \
           ${TROVE_IMAGE_OS} \
@@ -499,6 +499,10 @@ function create_guest_image {
         $TROVE_MANAGE db_load_datastore_config_parameters "$TROVE_DATASTORE_TYPE" "$TROVE_DATASTORE_VERSION" \
             $DEST/trove/trove/templates/$TROVE_DATASTORE_TYPE/validation-rules.json
     fi
+    # NOTE(wuchunyang): Create log directory so that guest agent can rsync logs to this directory
+    if [[ ${SYNC_LOG_TO_CONTROLLER} == "True" ]]; then
+    test -e /var/log/guest-agent-logs || sudo mkdir -p /var/log/guest-agent-logs/ && sudo chmod 777 /var/log/guest-agent-logs
+    fi
 }
 
 function create_registry_container {

integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/31-guest-agent-install

@@ -25,6 +25,12 @@ ${GUEST_VENV}/bin/pip install pip --upgrade
 ${GUEST_VENV}/bin/pip install -U -c /opt/upper-constraints.txt /opt/guest-agent
 chown -R ${GUEST_USERNAME}:root ${GUEST_VENV}
 
+# this should only happen in zuul system.
+if [[ ${SYNC_LOG_TO_CONTROLLER} == "True" ]]; then
+    sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;" ${SCRIPTDIR}/guest-log-collection.service > /etc/systemd/system/guest-log-collection.service
+    cp ${SCRIPTDIR}/guest-log-collection.timer /etc/systemd/system/guest-log-collection.timer
+fi
+
 if [[ ${DEV_MODE} == "true" ]]; then
     [[ -n "${HOST_SCP_USERNAME}" ]] || die "HOST_SCP_USERNAME needs to be set to the trovestack host user"
     [[ -n "${ESCAPED_PATH_TROVE}" ]] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the trovestack host"

integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-log-collection.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=OpenStack Trove Guest Log Collection Service
+
+[Service]
+Type=oneshot
+User=ubuntu
+Group=ubuntu
+
+EnvironmentFile=/etc/trove/controller.conf
+
+EnvironmentFile=/etc/trove/conf.d/guest_info.conf
+
+ExecStart=/bin/bash -c "sudo rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i /home/GUEST_USERNAME/.ssh/id_rsa' -az --exclude='.*' /var/log /var/lib/docker/containers HOST_SCP_USERNAME@$CONTROLLER:/var/log/guest-agent-logs/$guest_id"

integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-log-collection.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=OpenStack Trove Guest Log Collection Timer
+
+[Timer]
+OnActiveSec=10s
+OnUnitInactiveSec=10s
+Unit=guest-log-collection.service
+Persistent=true
+
+[Install]
+WantedBy=multi-user.target

integration/scripts/files/elements/guest-agent/post-install.d/31-enable-guest-agent-systemd

@@ -9,3 +9,7 @@ set -o pipefail
 if [ "$DIB_INIT_SYSTEM" == "systemd" ]; then
     systemctl enable $(svc-map guest-agent)
 fi
+
+if [[ ${SYNC_LOG_TO_CONTROLLER} == "True" ]]; then
+    systemctl enable guest-log-collection.timer
+fi

integration/scripts/functions_qemu

@@ -37,6 +37,11 @@ function build_guest_image() {
     if [[ "${DEV_MODE}" == "true" ]]; then
         export SSH_DIR=${SSH_DIR:-"$HOME/.ssh"}
         manage_ssh_keys
+        # sync_log_to_controller only works in dev mode
+        export SYNC_LOG_TO_CONTROLLER=${SYNC_LOG_TO_CONTROLLER:-False}
+    else
+        # set SYNC_LOG_TO_CONTROLLER to false when dev mode is false
+        export SYNC_LOG_TO_CONTROLLER=False
     fi
 
     TEMP=$(mktemp -d ${working_dir}/diskimage-create.XXXXXXX)

zuul.d/jobs.yaml

@@ -299,6 +299,7 @@
         Q_ML2_TENANT_NETWORK_TYPE: vxlan
         Q_ML2_PLUGIN_MECHANISM_DRIVERS: openvswitch
         TROVE_ENABLE_LOCAL_REGISTRY: True
+        SYNC_LOG_TO_CONTROLLER: True
         TROVE_DATASTORE_VERSION: 5.7.29
       devstack_local_conf:
         post-config:
@@ -307,6 +308,8 @@
               usage_timeout: 1800
         test-config:
           $TEMPEST_CONFIG:
+            auth:
+              tempest_roles: ResellerAdmin
             database:
               default_datastore_versions: mysql:5.7.29
       devstack_plugins:
@@ -330,6 +333,7 @@
         s-container: true
         s-object: true
         s-proxy: true
+        tls-proxy: false
         tempest: true
         q-svc: true
         q-agt: true
@@ -341,6 +345,8 @@
         ovn-northd: false
       tempest_test_regex: ^trove_tempest_plugin\.tests
       tempest_test_blacklist: '{{ ansible_user_dir }}/{{ zuul.projects["opendev.org/openstack/trove-tempest-plugin"].src_dir }}/tempest_blacklist.txt'
+      zuul_copy_output:
+        '/var/log/guest-agent-logs/': 'logs'
 
 - job:
     name: trove-tempest-postgres