Merge "First commit of devstack directory"

devstack/README.rst

@@ -0,0 +1,28 @@
+===============================
+Enabling Slogging in DevStack
+===============================
+
+1. Download Devstack::
+
+    git clone https://git.openstack.org/openstack-dev/devstack
+    cd devstack
+
+2. Add this repo as an external repository in ``local.conf`` file::
+
+    [[local|localrc]]
+    enable_plugin slogging https://git.openstack.org/openstack/slogging
+
+   To use stable branches, make sure devstack is on that branch, and specify
+   the branch name to enable_plugin, for example::
+
+    [[local|localrc]]
+    enable_plugin slogging https://git.openstack.org/openstack/slogging <refspec>
+
+   In case triggered by cron, set environ variable ``SLOGGING_CRON``::
+
+    [[local|localrc]]
+    SLOGGING_CRON=True
+
+3. Run ``stack.sh``::
+
+    ./stack.sh

devstack/files/cron.d/swift-access-log-uploader

@@ -0,0 +1 @@
+5 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-log-uploader %SLOGGING_BASE%/etc/log-processor.conf access

devstack/files/cron.d/swift-container-stats-log-creator

@@ -0,0 +1 @@
+5 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-container-stats-logger %SLOGGING_BASE%/etc/log-processor.conf 

devstack/files/cron.d/swift-stats-log-creator

@@ -0,0 +1 @@
+0 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-account-stats-logger %SLOGGING_BASE%/etc/log-processor.conf 

devstack/files/cron.d/swift-stats-log-uploader

@@ -0,0 +1,2 @@
+10 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-log-uploader %SLOGGING_BASE%/etc/log-processor.conf stats
+15 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-log-uploader %SLOGGING_BASE%/etc/log-processor.conf container-stats

devstack/files/cron.d/swift-stats-processor

@@ -0,0 +1 @@
+30 * * * * %STACK_USER% export PYTHONPATH=$PYTHONPATH:%SLOGGING_BASE%; %SLOGGING_BASE%/bin/swift-log-stats-collector %SLOGGING_BASE%/etc/log-processor.conf 

devstack/files/log-processor.conf

@@ -0,0 +1,50 @@
+[DEFAULT]
+swift_account = %TEMPACCOUNT%
+user = %STACK_USER%
+log_level = DEBUG
+log_facility = LOG_LOCAL1
+
+[log-processor]
+container_name = log_processing_data
+proxy_server_conf = /etc/swift/proxy-server.conf
+lookback_hours = 120
+lockback_window = 120
+format_type = json
+
+[log-processor-access]
+container_name = log_data
+log_dir = %DEST%/data/swift/logs/hourly/
+source_filename_pattern = ^
+    (?P<year>[0-9]{4})
+    (?P<month>[0-1][0-9])
+    (?P<day>[0-3][0-9])
+    (?P<hour>[0-2][0-9])
+    .*$
+class_path = slogging.access_processor.AccessLogProcessor
+unlink_log = True
+new_log_cutoff = 30
+
+[log-processor-stats]
+container_name = account_stats
+log_dir = %DEST%/data/swift/logs/stats/
+class_path = slogging.stats_processor.StatsLogProcessor
+log_name = stats-%Y%m%d%H_
+stats_type =  account
+account_server_data_dir = accounts
+devices = %DEST%/data/swift/1
+mount_check = false
+unlink_log = True
+new_log_cutoff = 30
+
+[log-processor-container-stats]
+container_name = container_stats
+log_dir = %DEST%/data/swift/logs/stats/
+class_path = slogging.stats_processor.StatsLogProcessor
+log_name = container-stats-%Y%m%d%H_
+stats_type = container
+container_server_data_dir = containers
+processable = false
+devices = %DEST%/data/swift/1
+mount_check = false
+unlink_log = True
+new_log_cutoff = 30

devstack/lib/slogging

@@ -0,0 +1,125 @@
+#!/bin/bash
+#
+# lib/slogging
+# Functions to control the configuration and operation of the slogging plugin
+
+# Dependencies:
+#
+# - Swift service has already installed into $DEST/swift with below in local.conf
+# - ``s-proxy``, ``s-container``, ``s-object``, ``s-object``
+
+# - ``functions`` file
+# - ``DEST``,  ``$STACK_USER``must be defined
+
+# - ``lib/swift`` file
+# - ``SWIFT_CONFIG_PROXY_SERVER`` must be defined
+
+# ``stack.sh`` file
+# - ``echo_summary`` function must be defined
+#
+
+# ``stack.sh`` calls the entry points in this order:
+#
+# - install_slogging
+# - configure_slogging
+# - verify_swift
+# - metering-sample_slogging
+
+function _get_token_and_endpoint {
+    curl -v -H "X-Storage-User:${SLOGGING['tempuser']}" \
+            -H "X-Storage-Pass:${SLOGGING['temppass']}" \
+               ${SLOGGING['auth_url']} 2>&1 | \
+    egrep 'X-Storage-Url:|X-Auth-Token:' | \
+    sed -e 's|: |=|g' -e 's|<||g' -e 's|\-|_|g' -e 's|\r||g'
+}
+
+function _exec_swift_api {
+    for envvar in $(set | awk -F'=' '/^OS_/{print $1}'); do
+        unset ${envvar}
+    done
+    swift -A ${SLOGGING['auth_url']} \
+          -U ${SLOGGING['tempuser']} \
+          -K ${SLOGGING['temppass']} stat
+}
+
+function install_slogging {
+    pushd $SLOGGING_BASE && {
+        sudo python setup.py install
+    } && popd
+}
+
+function configure_slogging {
+    local saio_base="$DEST/swift/doc/saio"
+    local rsyslog_conf="rsyslog.d/10-swift.conf"
+    local swift_log_base="$DEST/data/swift"
+
+    # Set config
+    if [ ! -f "/etc/$rsyslog_conf" ]; then
+        sudo cp $saio_base/$rsyslog_conf /etc/$rsyslog_conf
+    fi
+    sudo sed -e "
+        s|/var/log/swift|$swift_log_base/logs|g;
+        s|#\$templat|\$templat|g;
+        s|#local1\.|local1.|g;
+    " -i /etc/$rsyslog_conf
+    sudo sed -e '/PrivDropToGroup/c $PrivDropToGroup adm' /etc/rsyslog.conf
+
+    # Set directory
+    sudo mkdir -p $swift_log_base/logs/hourly
+    safe_chown -R $STACK_USER.adm $swift_log_base
+    safe_chmod 775 $swift_log_base/logs $swift_log_base/logs/hourly
+    add_user_to_group syslog adm
+    add_user_to_group $STACK_USER adm
+
+    # Reflect daemon
+    restart_service rsyslog
+
+    # Generate /etc/swift/log-processor.conf
+    local log_processor=$SLOGGING_BASE/etc/log-processor.conf
+    sudo cp $SLOGGING_BASE/devstack/files/log-processor.conf $log_processor
+
+    # set device
+    iniset $log_processor log-processor-container-stats devices $swift_log_base/1
+    iniset $log_processor log-processor-stats devices $swift_log_base/1
+
+    # set log_dir
+    iniset $log_processor log-processor-container-stats log_dir $swift_log_base/logs/stats/
+    iniset $log_processor log-processor-stats log_dir $swift_log_base/logs/stats/
+    iniset $log_processor log-processor-access log_dir $swift_log_base/logs/hourly/
+
+    iniset $log_processor DEFAULT swift_account ${SLOGGING['tempaccount']}
+    iniset $log_processor DEFAULT user $STACK_USER
+    iniset $SWIFT_CONFIG_PROXY_SERVER app:proxy-server log_facility LOG_LOCAL1
+
+    # Set cron
+    if [ "$SLOGGING_CRON" == "True" ]; then
+        sudo cp $SLOGGING_BASE/devstack/files/cron.d/swift-* /etc/cron.d/
+        sudo sed -e "
+            s|%STACK_USER%|$STACK_USER|g;
+            s|%SLOGGING_BASE%|$SLOGGING_BASE|g;
+        " -i /etc/cron.d/swift-*
+    else
+        sudo rm -f /etc/cron.d/swift-*
+    fi
+}
+
+function verify_swift {
+    # Internal proxy(identity) API
+    for var in $(_get_token_and_endpoint); do export $var; done
+    curl -v -H "X-Auth-Token: $X_Auth_Token" $X_Storage_Url
+
+    # Swift API
+    local retry=3
+    local interval=5
+    for i in {1..$retry}; do
+        ( _exec_swift_api ) && return 0
+        sleep $interval
+    done
+    return -1
+}
+
+function metering-sample_slogging {
+    echo_summary "Swift's Function Test"
+    $DEST/swift/.functests
+}
+

devstack/plugin.sh

@@ -0,0 +1,57 @@
+# plugin.sh - Devstack plugin.sh script to install and configure slogging with swift settings
+
+# Save trace setting
+XTRACE=$(set +o | grep xtrace)
+set -o xtrace
+
+echo_summary "slogging's plugin.sh was called..."
+source $DEST/slogging/devstack/lib/slogging
+
+# check for service enabled
+if is_plugin_enabled slogging; then
+
+    if [[ "$1" == "stack" && "$2" == "pre-install" ]]; then
+        # Set up system services                  # apt-get install, pip insall
+        #echo_summary "Configuring system services Slogging"
+        #install_package cowsay
+        :
+
+    elif [[ "$1" == "stack" && "$2" == "install" ]]; then
+        # Perform installation of service source  # python setup.py install
+        #echo_summary "Installing Slogging"
+        #install_slogging
+        :
+
+    elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
+        # Configure after the other layer 1 and 2 services have been configured
+        echo_summary "Configuring Slogging"  # memcached, rsyslog, ... etc
+        configure_slogging
+
+    elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
+        # Initialize and start the
+        echo_summary "Verifying Swift nomally installed"
+        verify_swift
+
+    elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
+        # Unit & Function test for slogging to get swift log
+        echo_summary "Metering Sample with Swift"
+        metering-sample_slogging
+    fi
+
+    if [[ "$1" == "unstack" ]]; then
+        # Shut down template services
+        # no-op
+        :
+    fi
+
+    if [[ "$1" == "clean" ]]; then
+        # Remove state and transient data
+        # Remember clean.sh first calls unstack.sh
+        # no-op
+        :
+    fi
+fi
+
+# Restore
+$XTRACE
+

devstack/settings

@@ -0,0 +1,9 @@
+# settings file for sloggin plugin
+declare -A -g SLOGGING
+SLOGGING['tempuser']="test:tester"
+SLOGGING['temppass']="testing"
+SLOGGING['tempaccount']="TEMPAUTH_test"
+SLOGGING['auth_url']="http://127.0.0.1:8080/auth/v1.0"
+SLOGGING_BASE=$DEST/slogging
+
+enable_service s-proxy s-object s-container s-account