From 48b3856e2eb90a024a6659a6ea812d4e3a8d4bfd Mon Sep 17 00:00:00 2001 From: Hajime Kondo Date: Fri, 27 Apr 2018 09:01:23 +0000 Subject: [PATCH] First commit of devstack directory Add devstack files and directories below. - Document: README.rst - Entrypoint: plugin.sh - Functions Library: lib/slogging - Variable Settings: settings - Configuration: files/log-processor.conf - Cron Settings: files/cron.d/* Change-Id: I98df6519ede6916e41579b2318bdc71c2b15df94 --- devstack/README.rst | 28 ++++ .../files/cron.d/swift-access-log-uploader | 1 + .../cron.d/swift-container-stats-log-creator | 1 + devstack/files/cron.d/swift-stats-log-creator | 1 + .../files/cron.d/swift-stats-log-uploader | 2 + devstack/files/cron.d/swift-stats-processor | 1 + devstack/files/log-processor.conf | 50 +++++++ devstack/lib/slogging | 125 ++++++++++++++++++ devstack/plugin.sh | 57 ++++++++ devstack/settings | 9 ++ 10 files changed, 275 insertions(+) create mode 100644 devstack/README.rst create mode 100644 devstack/files/cron.d/swift-access-log-uploader create mode 100644 devstack/files/cron.d/swift-container-stats-log-creator create mode 100644 devstack/files/cron.d/swift-stats-log-creator create mode 100644 devstack/files/cron.d/swift-stats-log-uploader create mode 100644 devstack/files/cron.d/swift-stats-processor create mode 100644 devstack/files/log-processor.conf create mode 100644 devstack/lib/slogging create mode 100644 devstack/plugin.sh create mode 100644 devstack/settings diff --git a/devstack/README.rst b/devstack/README.rst new file mode 100644 index 0000000..f3f9b29 --- /dev/null +++ b/devstack/README.rst @@ -0,0 +1,28 @@ +=============================== +Enabling Slogging in DevStack +=============================== + +1. Download Devstack:: + + git clone https://git.openstack.org/openstack-dev/devstack + cd devstack + +2. Add this repo as an external repository in ``local.conf`` file:: + + [[local|localrc]] + enable_plugin slogging https://git.openstack.org/openstack/slogging + + To use stable branches, make sure devstack is on that branch, and specify + the branch name to enable_plugin, for example:: + + [[local|localrc]] + enable_plugin slogging https://git.openstack.org/openstack/slogging stable/juno + + In case triggered by cron, set environ variable ``SLOGGING_CRON``:: + + [[local|localrc]] + SLOGGING_CRON=True + +3. Run ``stack.sh``:: + + ./stack.sh diff --git a/devstack/files/cron.d/swift-access-log-uploader b/devstack/files/cron.d/swift-access-log-uploader new file mode 100644 index 0000000..973096b --- /dev/null +++ b/devstack/files/cron.d/swift-access-log-uploader @@ -0,0 +1 @@ +5 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-log-uploader %SLOGGING_BASE%/etc/log-processor.conf access diff --git a/devstack/files/cron.d/swift-container-stats-log-creator b/devstack/files/cron.d/swift-container-stats-log-creator new file mode 100644 index 0000000..f8f3d5b --- /dev/null +++ b/devstack/files/cron.d/swift-container-stats-log-creator @@ -0,0 +1 @@ +5 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-container-stats-logger %SLOGGING_BASE%/etc/log-processor.conf diff --git a/devstack/files/cron.d/swift-stats-log-creator b/devstack/files/cron.d/swift-stats-log-creator new file mode 100644 index 0000000..b4627b7 --- /dev/null +++ b/devstack/files/cron.d/swift-stats-log-creator @@ -0,0 +1 @@ +0 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-account-stats-logger %SLOGGING_BASE%/etc/log-processor.conf diff --git a/devstack/files/cron.d/swift-stats-log-uploader b/devstack/files/cron.d/swift-stats-log-uploader new file mode 100644 index 0000000..cd0c400 --- /dev/null +++ b/devstack/files/cron.d/swift-stats-log-uploader @@ -0,0 +1,2 @@ +10 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-log-uploader %SLOGGING_BASE%/etc/log-processor.conf stats +15 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-log-uploader %SLOGGING_BASE%/etc/log-processor.conf container-stats diff --git a/devstack/files/cron.d/swift-stats-processor b/devstack/files/cron.d/swift-stats-processor new file mode 100644 index 0000000..3ad5693 --- /dev/null +++ b/devstack/files/cron.d/swift-stats-processor @@ -0,0 +1 @@ +30 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-log-stats-collector %SLOGGING_BASE%/etc/log-processor.conf diff --git a/devstack/files/log-processor.conf b/devstack/files/log-processor.conf new file mode 100644 index 0000000..1e8d59b --- /dev/null +++ b/devstack/files/log-processor.conf @@ -0,0 +1,50 @@ +[DEFAULT] +swift_account = %TEMPACCOUNT% +user = %STACK_USER% +log_level = DEBUG +log_facility = LOG_LOCAL1 + +[log-processor] +container_name = log_processing_data +proxy_server_conf = /etc/swift/proxy-server.conf +lookback_hours = 120 +lockback_window = 120 +format_type = json + +[log-processor-access] +container_name = log_data +log_dir = %DEST%/data/swift/logs/hourly/ +source_filename_pattern = ^ + (?P[0-9]{4}) + (?P[0-1][0-9]) + (?P[0-3][0-9]) + (?P[0-2][0-9]) + .*$ +class_path = slogging.access_processor.AccessLogProcessor +unlink_log = True +new_log_cutoff = 30 + +[log-processor-stats] +container_name = account_stats +log_dir = %DEST%/data/swift/logs/stats/ +class_path = slogging.stats_processor.StatsLogProcessor +log_name = stats-%Y%m%d%H_ +stats_type = account +account_server_data_dir = accounts +devices = %DEST%/data/swift/1 +mount_check = false +unlink_log = True +new_log_cutoff = 30 + +[log-processor-container-stats] +container_name = container_stats +log_dir = %DEST%/data/swift/logs/stats/ +class_path = slogging.stats_processor.StatsLogProcessor +log_name = container-stats-%Y%m%d%H_ +stats_type = container +container_server_data_dir = containers +processable = false +devices = %DEST%/data/swift/1 +mount_check = false +unlink_log = True +new_log_cutoff = 30 diff --git a/devstack/lib/slogging b/devstack/lib/slogging new file mode 100644 index 0000000..d1666b2 --- /dev/null +++ b/devstack/lib/slogging @@ -0,0 +1,125 @@ +#!/bin/bash +# +# lib/slogging +# Functions to control the configuration and operation of the slogging plugin + +# Dependencies: +# +# - Swift service has already installed into $DEST/swift with below in local.conf +# - ``s-proxy``, ``s-container``, ``s-object``, ``s-object`` + +# - ``functions`` file +# - ``DEST``, ``$STACK_USER``must be defined + +# - ``lib/swift`` file +# - ``SWIFT_CONFIG_PROXY_SERVER`` must be defined + +# ``stack.sh`` file +# - ``echo_summary`` function must be defined +# + +# ``stack.sh`` calls the entry points in this order: +# +# - install_slogging +# - configure_slogging +# - verify_swift +# - metering-sample_slogging + +function _get_token_and_endpoint { + curl -v -H "X-Storage-User:${SLOGGING['tempuser']}" \ + -H "X-Storage-Pass:${SLOGGING['temppass']}" \ + ${SLOGGING['auth_url']} 2>&1 | \ + egrep 'X-Storage-Url:|X-Auth-Token:' | \ + sed -e 's|: |=|g' -e 's|<||g' -e 's|\-|_|g' -e 's|\r||g' +} + +function _exec_swift_api { + for envvar in $(set | awk -F'=' '/^OS_/{print $1}'); do + unset ${envvar} + done + swift -A ${SLOGGING['auth_url']} \ + -U ${SLOGGING['tempuser']} \ + -K ${SLOGGING['temppass']} stat +} + +function install_slogging { + pushd $SLOGGING_BASE && { + sudo python setup.py install + } && popd +} + +function configure_slogging { + local saio_base="$DEST/swift/doc/saio" + local rsyslog_conf="rsyslog.d/10-swift.conf" + local swift_log_base="$DEST/data/swift" + + # Set config + if [ ! -f "/etc/$rsyslog_conf" ]; then + sudo cp $saio_base/$rsyslog_conf /etc/$rsyslog_conf + fi + sudo sed -e " + s|/var/log/swift|$swift_log_base/logs|g; + s|#\$templat|\$templat|g; + s|#local1\.|local1.|g; + " -i /etc/$rsyslog_conf + sudo sed -e '/PrivDropToGroup/c $PrivDropToGroup adm' /etc/rsyslog.conf + + # Set directory + sudo mkdir -p $swift_log_base/logs/hourly + safe_chown -R $STACK_USER.adm $swift_log_base + safe_chmod 775 $swift_log_base/logs $swift_log_base/logs/hourly + add_user_to_group syslog adm + add_user_to_group $STACK_USER adm + + # Reflect daemon + restart_service rsyslog + + # Generate /etc/swift/log-processor.conf + local log_processor=$SLOGGING_BASE/etc/log-processor.conf + sudo cp $SLOGGING_BASE/devstack/files/log-processor.conf $log_processor + + # set device + iniset $log_processor log-processor-container-stats devices $swift_log_base/1 + iniset $log_processor log-processor-stats devices $swift_log_base/1 + + # set log_dir + iniset $log_processor log-processor-container-stats log_dir $swift_log_base/logs/stats/ + iniset $log_processor log-processor-stats log_dir $swift_log_base/logs/stats/ + iniset $log_processor log-processor-access log_dir $swift_log_base/logs/hourly/ + + iniset $log_processor DEFAULT swift_account ${SLOGGING['tempaccount']} + iniset $log_processor DEFAULT user $STACK_USER + iniset $SWIFT_CONFIG_PROXY_SERVER app:proxy-server log_facility LOG_LOCAL1 + + # Set cron + if [ "$SLOGGING_CRON" == "True" ]; then + sudo cp $SLOGGING_BASE/devstack/files/cron.d/swift-* /etc/cron.d/ + sudo sed -e " + s|%STACK_USER%|$STACK_USER|g; + s|%SLOGGING_BASE%|$SLOGGING_BASE|g; + " -i /etc/cron.d/swift-* + else + sudo rm -f /etc/cron.d/swift-* + fi +} + +function verify_swift { + # Internal proxy(identity) API + for var in $(_get_token_and_endpoint); do export $var; done + curl -v -H "X-Auth-Token: $X_Auth_Token" $X_Storage_Url + + # Swift API + local retry=3 + local interval=5 + for i in {1..$retry}; do + ( _exec_swift_api ) && return 0 + sleep $interval + done + return -1 +} + +function metering-sample_slogging { + echo_summary "Swift's Function Test" + $DEST/swift/.functests +} + diff --git a/devstack/plugin.sh b/devstack/plugin.sh new file mode 100644 index 0000000..c6982ba --- /dev/null +++ b/devstack/plugin.sh @@ -0,0 +1,57 @@ +# plugin.sh - Devstack plugin.sh script to install and configure slogging with swift settings + +# Save trace setting +XTRACE=$(set +o | grep xtrace) +set -o xtrace + +echo_summary "slogging's plugin.sh was called..." +source $DEST/slogging/devstack/lib/slogging + +# check for service enabled +if is_plugin_enabled slogging; then + + if [[ "$1" == "stack" && "$2" == "pre-install" ]]; then + # Set up system services # apt-get install, pip insall + #echo_summary "Configuring system services Slogging" + #install_package cowsay + : + + elif [[ "$1" == "stack" && "$2" == "install" ]]; then + # Perform installation of service source # python setup.py install + #echo_summary "Installing Slogging" + #install_slogging + : + + elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then + # Configure after the other layer 1 and 2 services have been configured + echo_summary "Configuring Slogging" # memcached, rsyslog, ... etc + configure_slogging + + elif [[ "$1" == "stack" && "$2" == "extra" ]]; then + # Initialize and start the + echo_summary "Verifying Swift nomally installed" + verify_swift + + elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then + # Unit & Function test for slogging to get swift log + echo_summary "Metering Sample with Swift" + metering-sample_slogging + fi + + if [[ "$1" == "unstack" ]]; then + # Shut down template services + # no-op + : + fi + + if [[ "$1" == "clean" ]]; then + # Remove state and transient data + # Remember clean.sh first calls unstack.sh + # no-op + : + fi +fi + +# Restore +$XTRACE + diff --git a/devstack/settings b/devstack/settings new file mode 100644 index 0000000..e596409 --- /dev/null +++ b/devstack/settings @@ -0,0 +1,9 @@ +# settings file for sloggin plugin +declare -A -g SLOGGING +SLOGGING['tempuser']="test:tester" +SLOGGING['temppass']="testing" +SLOGGING['tempaccount']="TEMPAUTH_test" +SLOGGING['auth_url']="http://127.0.0.1:8080/auth/v1.0" +SLOGGING_BASE=$DEST/slogging + +enable_service s-proxy s-object s-container s-account