fix for upstream repository of slogging

.gitignore

@@ -4,3 +4,5 @@ doc/build/*
 dist
 .coverage
 deb_dist
+.tox
+.idea

CONTRIBUTING.rst

@@ -0,0 +1,20 @@
+Contributing to slogging
+========================
+
+If you would like to contribute to the development of OpenStack, you must
+follow the steps in this page:
+
+   http://docs.openstack.org/infra/manual/developers.html
+
+If you already have a good understanding of how the system works and your
+OpenStack accounts are set up, you can skip to the development workflow
+section of this documentation to learn how changes to OpenStack should be
+submitted for review via the Gerrit tool:
+
+   http://docs.openstack.org/infra/manual/developers.html#development-workflow
+
+Pull requests submitted through GitHub will be ignored.
+
+Bugs should be filed on Launchpad, not GitHub:
+
+   https://bugs.launchpad.net/../horizon

README.rst

@@ -1,17 +1,16 @@
-Swift stats system
-==================
+====================
+Welcome to slogging!
+====================
 
-The swift stats system is composed of three parts parts: log creation, log
-uploading, and log processing. The system handles two types of logs (access
-and account stats), but it can be extended to handle other types of logs.
+slogging is a stats and logging tools for OpenStack Swift.
 
-How to Build to Debian Packages
-===============================
+* License: Apache license
+* Documentation: https://docs.openstack.org/slogging/latest/
+* Source: https://git.openstack.org/cgit/openstack/slogging
+* Bugs: https://bugs.launchpad.net/slogging
 
-Make sure you have python-stdeb installed first:
+Team and repository tags
+------------------------
 
-    sudo apt-get install python-stdeb
-
-then:
-
-    python setup.py --command-packages=stdeb.command bdist_deb
+.. image:: https://governance.openstack.org/tc/badges/slogging.svg
+    :target: http://governance.openstack.org/reference/tags/index.html

babel.cfg

@@ -0,0 +1,2 @@
+[python: **.py]
+

bin/swift-access-log-delivery

@@ -15,8 +15,8 @@
 # limitations under the License.
 
 from slogging.access_log_delivery import AccessLogDeliveryDaemon
-from swift.common.utils import parse_options
 from swift.common.daemon import run_daemon
+from swift.common.utils import parse_options
 
 if __name__ == '__main__':
     conf_file, options = parse_options(once=True)

bin/swift-account-stats-logger

@@ -15,8 +15,8 @@
 # limitations under the License.
 
 from slogging.db_stats_collector import AccountStatsCollector
-from swift.common.utils import parse_options
 from swift.common.daemon import run_daemon
+from swift.common.utils import parse_options
 
 if __name__ == '__main__':
     conf_file, options = parse_options()

bin/swift-container-stats-logger

@@ -15,8 +15,8 @@
 # limitations under the License.
 
 from slogging.db_stats_collector import ContainerStatsCollector
-from swift.common.utils import parse_options
 from swift.common.daemon import run_daemon
+from swift.common.utils import parse_options
 
 if __name__ == '__main__':
     conf_file, options = parse_options()

bin/swift-log-stats-collector

@@ -17,16 +17,17 @@
 from optparse import OptionParser
 
 from slogging.log_processor import LogProcessorDaemon
-from swift.common.utils import parse_options
 from swift.common.daemon import run_daemon
+from swift.common.utils import parse_options
 
 if __name__ == '__main__':
     parser = OptionParser(usage='Usage: %prog [options] <conf_file>')
 
     parser.add_option('--lookback_hours', type='int', dest='lookback_hours',
-        help='Hours in the past to start looking for log files')
+                      help='Hours in the past to start looking for log files')
     parser.add_option('--lookback_window', type='int', dest='lookback_window',
-        help='Hours past lookback_hours to stop looking for log files')
+                      help='Hours past lookback_hours '
+                           'to stop looking for log files')
 
     conf_file, options = parse_options(parser)
     # currently the LogProcessorDaemon only supports run_once

bin/swift-log-uploader

@@ -14,11 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
 from optparse import OptionParser
 from slogging.log_uploader import LogUploader
-from swift.common.utils import parse_options
 from swift.common import utils
+from swift.common.utils import parse_options
+import sys
 
 if __name__ == '__main__':
     parser = OptionParser("Usage: %prog CONFIG_FILE PLUGIN")
@@ -30,7 +30,7 @@ if __name__ == '__main__':
     try:
         plugin = options['extra_args'][0]
     except (IndexError, KeyError):
-        print "Error: missing plugin name"
+        print("Error: missing plugin name")
         sys.exit(1)
 
     uploader_conf = utils.readconf(conf_file, 'log-processor')

doc/requirements.txt

@@ -0,0 +1,8 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+
+openstackdocstheme>=1.18.1 # Apache-2.0
+sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD
+reno>=2.5.0 # Apache-2.0
+sphinxcontrib-httpdomain>=1.3.0 # BSD

doc/source/conf.py

@@ -1,9 +1,24 @@
 # -*- coding: utf-8 -*-
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# slogging documentation build configuration file, created by
-# sphinx-quickstart on Fri Jun 17 13:32:10 2011.
+#    http://www.apache.org/licenses/LICENSE-2.0
 #
-# This file is execfile()d with the current directory set to its containing dir.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright (c) 2010-2012 OpenStack Foundation.
+#
+# Swift documentation build configuration file, created by
+# sphinx-quickstart on Tue May 18 13:50:15 2010.
+#
+# This file is execfile()d with the current directory set to its containing
+# dir.
 #
 # Note that not all possible configuration values are present in this
 # autogenerated file.
@@ -11,184 +26,208 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import sys, os
+import datetime
+import logging
+import os
+# from slogging import __version__
+import sys
+
+# NOTE(amotoki): Our current doc build job uses an older version of
+# liberasurecode which comes from Ubuntu 16.04.
+# pyeclib emits a warning message if liberasurecode <1.3.1 is used [1] and
+# this causes the doc build failure if warning-is-error is enabled in Sphinx.
+# As a workaround we suppress the warning message from pyeclib until we use
+# a newer version of liberasurecode in our doc build job.
+# [1] https://github.com/openstack/pyeclib/commit/d163972b
+logging.getLogger('pyeclib').setLevel(logging.ERROR)
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.append(os.path.abspath('.'))
+sys.path.extend([os.path.abspath('../slogging'), os.path.abspath('..'),
+                 os.path.abspath('../bin')])
 
-# -- General configuration -----------------------------------------------------
+# -- General configuration ----------------------------------------------------
 
-# Add any Sphinx extension module names here, as strings. They can be extensions
-# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = []
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ['sphinx.ext.autodoc',
+              'sphinx.ext.todo',
+              'sphinx.ext.coverage',
+              'sphinx.ext.ifconfig',
+              'openstackdocstheme']
+todo_include_todos = True
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+# templates_path = []
 
 # The suffix of source filenames.
 source_suffix = '.rst'
 
 # The encoding of source files.
-#source_encoding = 'utf-8'
+# source_encoding = 'utf-8'
 
 # The master toctree document.
 master_doc = 'index'
 
 # General information about the project.
-project = u'slogging'
-copyright = u'2011, Openstack, LLC'
-
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
-version = '1.0'
-# The full version, including alpha/beta/rc tags.
-release = '1.0'
+project = u'Slogging'
+copyright = u'%d, OpenStack Foundation' % datetime.datetime.now().year
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
-#language = None
+# language = None
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-#today = ''
+# today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'
 
 # List of documents that shouldn't be included in the build.
-#unused_docs = []
+# unused_docs = []
 
 # List of directories, relative to source directory, that shouldn't be searched
 # for source files.
 exclude_trees = []
 
-# The reST default role (used for this markup: `text`) to use for all documents.
-#default_role = None
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+# default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
+# add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
-#add_module_names = True
+# add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
-#show_authors = False
+show_authors = True
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'
 
 # A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+modindex_common_prefix = ['slogging.']
 
 
-# -- Options for HTML output ---------------------------------------------------
+# -- Options for HTML output -----------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  Major themes that come with
 # Sphinx are currently 'default' and 'sphinxdoc'.
-html_theme = 'default'
+# html_theme = 'default'
+# html_theme_path = ["."]
+html_theme = 'openstackdocs'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-#html_theme_options = {}
+# html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+# html_theme_path = []
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
-#html_title = None
+# html_title = None
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-#html_logo = None
+# html_logo = None
 
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-#html_favicon = None
+# html_favicon = None
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+# html_static_path = ['_static']
+
+# Add any paths that contain "extra" files, such as .htaccess or
+# robots.txt.
+# html_extra_path = ['_extra']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+# html_last_updated_fmt = '%b %d, %Y'
+html_last_updated_fmt = '%Y-%m-%d %H:%M'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
-#html_use_smartypants = True
+# html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+# html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-#html_additional_pages = {}
+# html_additional_pages = {}
 
 # If false, no module index is generated.
-#html_use_modindex = True
+# html_use_modindex = True
 
 # If false, no index is generated.
-#html_use_index = True
+# html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
+# html_show_sourcelink = True
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
-#html_use_opensearch = ''
+# html_use_opensearch = ''
 
 # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = ''
+# html_file_suffix = ''
 
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'sloggingdoc'
 
 
-# -- Options for LaTeX output --------------------------------------------------
+# -- Options for LaTeX output -------------------------------------------------
 
 # The paper size ('letter' or 'a4').
-#latex_paper_size = 'letter'
+# latex_paper_size = 'letter'
 
 # The font size ('10pt', '11pt' or '12pt').
-#latex_font_size = '10pt'
+# latex_font_size = '10pt'
 
 # Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title, author, documentclass [howto/manual]).
+# (source start file, target name, title, author, documentclass
+# [howto/manual]).
 latex_documents = [
-  ('index', 'slogging.tex', u'slogging Documentation',
-   u'Openstack, LLC', 'manual'),
+    ('index', 'Slogging.tex', u'Slogging Documentation',
+     u'Slogging Team', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-#latex_logo = None
+# latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
-#latex_use_parts = False
+# latex_use_parts = False
 
 # Additional stuff for the LaTeX preamble.
-#latex_preamble = ''
+# latex_preamble = ''
 
 # Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []
 
 # If false, no module index is generated.
-#latex_use_modindex = True
+# latex_use_modindex = True
+
+# -- Options for openstackdocstheme -------------------------------------------
+repository_name = 'openstack/slogging'
+bug_project = 'slogging'
+bug_tag = ''

doc/source/contributor/contributing.rst

@@ -0,0 +1,41 @@
+=================
+How to Contribute
+=================
+
+Contributor License Agreement
+-----------------------------
+
+.. index::
+   single: license; agreement
+
+In order to contribute to the slogging project, you need to have
+signed OpenStack's contributor's agreement.
+
+.. seealso::
+
+   * http://docs.openstack.org/infra/manual/developers.html
+   * http://wiki.openstack.org/CLA
+
+LaunchPad Project
+-----------------
+
+Most of the tools used for OpenStack depend on a launchpad.net ID for
+authentication.
+
+.. seealso::
+
+   * https://launchpad.net
+   * https://launchpad.net/slogging
+
+Project Hosting Details
+-------------------------
+
+Bug tracker
+    http://launchpad.net/slogging
+
+Code Hosting
+    https://git.openstack.org/cgit/openstack/slogging
+
+Code Review
+    https://review.openstack.org/#/q/status:open+project:openstack/slogging,n,z
+

doc/source/contributor/index.rst

@@ -0,0 +1,8 @@
+===========================
+ Contributor Documentation
+===========================
+
+.. toctree::
+   :maxdepth: 2
+
+   contributing

doc/source/index.rst

@@ -1,23 +1,33 @@
-.. slogging documentation master file, created by
-   sphinx-quickstart on Fri Jun 17 13:32:10 2011.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
+..
+      Copyright 2010-2012 OpenStack Foundation
+      All Rights Reserved.
 
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+====================================
 Welcome to slogging's documentation!
 ====================================
 
-Contents:
+slogging is a stats and logging tools for OpenStack Swift which is
+composed of three parts parts: log creation, log uploading, and log processing.
+
+The system handles two types of logs (access and account stats),
+but it can be extended to handle other types of logs.
+
 
 .. toctree::
-   :maxdepth: 2
-
-   license
-   overview_stats
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
+   :maxdepth: 1
 
+   user/index
+   install/index
+   contributor/index

doc/source/install/build_debian_packages.rst

@@ -0,0 +1,20 @@
+===============================
+How to Build to Debian Packages
+===============================
+
+#. Make sure you have python-stdeb installed::
+
+    sudo apt-get install python-stdeb
+
+#. Also make sure pbr is installed as python package::
+
+    pip install pbr
+
+#. Then type following command at the top of slogging directory::
+
+    python setup.py --command-packages=stdeb.command bdist_deb
+
+#. Check if python-slogging package is successfully created::
+
+    ls deb_dist/python-slogging-[slogging-version]*
+

doc/source/install/build_rpm_packages.rst

@@ -0,0 +1,15 @@
+============================
+How to Build to RPM Packages
+============================
+
+#. Make sure you have rpm-build installed::
+
+    sudo yum install rpm-build
+
+#. Thsn type following command at the top of slogging directory::
+
+    sudo python setup.py bdist_rpm
+
+#. Check if the RPM package has built::
+
+    ls dist/slogging-[slogging-version]

doc/source/install/index.rst

@@ -0,0 +1,12 @@
+=============
+Install Guide
+=============
+
+.. toctree::
+   :maxdepth: 1
+
+   build_debian_packages
+   install_debian_packages
+   build_rpm_packages
+   install_rpm_packages
+   run_slogging_on_saio

doc/source/install/install_debian_packages.rst

@@ -0,0 +1,41 @@
+===================================
+How to Install with Debian Packages
+===================================
+
+#. Install Debian Package::
+
+    sudo dpkg -i python-slogging_[slogging-version]_all.deb
+
+#. You can ignore following kind of error messages.::
+
+    dpkg: dependency problems prevent configuration of python-slogging:
+     python-slogging depends on python-pbr; however:
+      Package python-pbr is not installed.
+     python-slogging depends on python-iptools; however:
+      Package python-iptools is not installed.
+     python-slogging depends on python-tzlocal; however:
+      Package python-tzlocal is not installed.
+     python-slogging depends on python-swift; however:
+      Package python-swift is not installed.
+
+    dpkg: error processing package python-slogging (--install):
+     dependency problems - leaving unconfigured
+    Errors were encountered while processing:
+     python-slogging
+
+#. Check if the Debian Package has successfully installed::
+
+    dpkg -l | grep slogging
+
+#. After install Debian packages, you need to install following dependent packages::
+
+    pbr
+    iptools
+    tzlocal
+    swift
+
+- You can install Swift by `SAIO - Swift All In One <https://docs.openstack.org/swift/latest/development_saio.html>`_.
+- You can install pbr, iptools, tzlocal by pip command like::
+
+    pip install [package_name]
+

doc/source/install/install_rpm_packages.rst

@@ -0,0 +1,12 @@
+================================
+How to Install with RPM Packages
+================================
+
+#. Install RPM Package::
+
+    sudo rpm -ivh slogging-[slogging-version].noarch.rpm
+
+#. Check if the RPM Package has successfully installed::
+
+    sudo rpm -qa | grep slogging
+

doc/source/install/run_slogging_on_saio.rst

@@ -0,0 +1,112 @@
+==============================================
+Running the slogging on SAIO(Swift All In One)
+==============================================
+
+This page shows you how to install slogging on SAIO(Swift All In One)
+environment.
+
+
+#. Create a swift account to use for storing stats information, and note the
+   account hash. The hash will be used in config files.
+
+#. Edit ``/etc/rsyslog.d/10-swift.conf``::
+
+    # Uncomment the following to have a log containing all logs together
+    #local1,local2,local3,local4,local5.*   /var/log/swift/all.log
+
+    $template HourlyProxyLog,"/var/log/swift/hourly/%$YEAR%%$MONTH%%$DAY%%$HOUR%"
+    local1.*;local1.!notice ?HourlyProxyLog
+
+    local1.*;local1.!notice /var/log/swift/proxy.log
+    local1.notice           /var/log/swift/proxy.error
+    local1.*                ~
+
+#. Edit ``/etc/rsyslog.conf`` and make the following change::
+
+    $PrivDropToGroup adm
+
+#. ``mkdir -p /var/log/swift/hourly``
+#. ``chown -R syslog.adm /var/log/swift``
+#. ``chmod 775 /var/log/swift /var/log/swift/hourly``
+#. ``service rsyslog restart``
+#. ``usermod -a -G adm <your-user-name>``
+#. Relogin to let the group change take effect.
+#. Create ``/etc/swift/log-processor.conf``::
+
+    [log-processor]
+    swift_account = <your-stats-account-hash>
+    user = <your-user-name>
+
+    [log-processor-access]
+    swift_account = <your-stats-account-hash>
+    container_name = log_data
+    log_dir = /var/log/swift/hourly/
+    source_filename_pattern = ^
+        (?P<year>[0-9]{4})
+        (?P<month>[0-1][0-9])
+        (?P<day>[0-3][0-9])
+        (?P<hour>[0-2][0-9])
+        .*$
+    class_path = slogging.access_processor.AccessLogProcessor
+    user = <your-user-name>
+
+    [log-processor-stats]
+    swift_account = <your-stats-account-hash>
+    container_name = account_stats
+    log_dir = /var/log/swift/stats/
+    class_path = slogging.stats_processor.StatsLogProcessor
+    devices = /srv/1/node
+    mount_check = false
+    user = <your-user-name>
+
+    [log-processor-container-stats]
+    swift_account = <your-stats-account-hash>
+    container_name = container_stats
+    log_dir = /var/log/swift/stats/
+    class_path = slogging.stats_processor.StatsLogProcessor
+    processable = false
+    devices = /srv/1/node
+    mount_check = false
+    user = <your-user-name>
+
+#. Add the following under [app:proxy-server] in ``/etc/swift/proxy-server.conf``::
+
+    log_facility = LOG_LOCAL1
+
+#. Create a ``cron`` job to run once per hour to create the stats logs. In
+   ``/etc/cron.d/swift-stats-log-creator``::
+
+    0 * * * * <your-user-name> /usr/local/bin/swift-account-stats-logger /etc/swift/log-processor.conf
+
+#. Create a ``cron`` job to run once per hour to create the container stats logs. In
+   ``/etc/cron.d/swift-container-stats-log-creator``::
+
+    5 * * * * <your-user-name> /usr/local/bin/swift-container-stats-logger /etc/swift/log-processor.conf
+
+#. Create a ``cron`` job to run once per hour to upload the stats logs. In
+   ``/etc/cron.d/swift-stats-log-uploader``::
+
+    10 * * * * <your-user-name> /usr/local/bin/swift-log-uploader /etc/swift/log-processor.conf stats
+
+#. Create a ``cron`` job to run once per hour to upload the stats logs. In
+   ``/etc/cron.d/swift-stats-log-uploader``::
+
+    15 * * * * <your-user-name> /usr/local/bin/swift-log-uploader /etc/swift/log-processor.conf container-stats
+
+#. Create a ``cron`` job to run once per hour to upload the access logs. In
+   ``/etc/cron.d/swift-access-log-uploader``::
+
+    5 * * * * <your-user-name> /usr/local/bin/swift-log-uploader /etc/swift/log-processor.conf access
+
+#. Create a ``cron`` job to run once per hour to process the logs. In
+   ``/etc/cron.d/swift-stats-processor``::
+
+    30 * * * * <your-user-name> /usr/local/bin/swift-log-stats-collector /etc/swift/log-processor.conf
+
+After running for a few hours, you should start to see .csv files in the
+``log_processing_data`` container in the swift stats account that was created
+earlier. This file will have one entry per account per hour for each account
+with activity in that hour. One .csv file should be produced per hour. Note
+that the stats will be delayed by at least two hours by default. This can be
+changed with the ``new_log_cutoff`` variable in the config file. See
+``log-processor.conf-sample`` for more details.

doc/source/license.rst

@@ -1,225 +0,0 @@
-.. _license:
-
-*******
-LICENSE
-*******
-
-::
-
-    Copyright (c) 2010-2011 OpenStack, LLC
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-    implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

doc/source/overview_stats.rst

@@ -1,192 +0,0 @@
-==================
-Swift stats system
-==================
-
-The swift stats system is composed of three parts parts: log creation, log
-uploading, and log processing. The system handles two types of logs (access
-and account stats), but it can be extended to handle other types of logs.
-
----------
-Log Types
----------
-
-***********
-Access logs
-***********
-
-Access logs are the proxy server logs. Rackspace uses syslog-ng to redirect
-the proxy log output to an hourly log file. For example, a proxy request that
-is made on August 4, 2010 at 12:37 gets logged in a file named 2010080412.
-This allows easy log rotation and easy per-hour log processing.
-
-*********************************
-Account / Container DB stats logs
-*********************************
-
-DB stats logs are generated by a stats system process.
-swift-account-stats-logger runs on each account server (via cron) and walks
-the filesystem looking for account databases. When an account database is
-found, the logger selects the account hash, bytes_used, container_count, and
-object_count. These values are then written out as one line in a csv file. One
-csv file is produced for every run of swift-account-stats-logger. This means
-that, system wide, one csv file is produced for every storage node. Rackspace
-runs the account stats logger every hour. Therefore, in a cluster of ten
-account servers, ten csv files are produced every hour. Also, every account
-will have one entry for every replica in the system. On average, there will be
-three copies of each account in the aggregate of all account stat csv files
-created in one system-wide run. The swift-container-stats-logger runs in a
-similar fashion, scanning the container dbs.
-
-----------------------
-Log Processing plugins
-----------------------
-
-The swift stats system is written to allow a plugin to be defined for every
-log type. Swift includes plugins for both access logs and storage stats logs.
-Each plugin is responsible for defining, in a config section, where the logs
-are stored on disk, where the logs will be stored in swift (account and
-container), the filename format of the logs on disk, the location of the
-plugin class definition, and any plugin-specific config values.
-
-The plugin class definition defines three methods. The constructor must accept
-one argument (the dict representation of the plugin's config section). The
-process method must accept an iterator, and the account, container, and object
-name of the log. The keylist_mapping accepts no parameters.
-
--------------
-Log Uploading
--------------
-
-swift-log-uploader accepts a config file and a plugin name. It finds the log
-files on disk according to the plugin config section and uploads them to the
-swift cluster. This means one uploader process will run on each proxy server
-node and each account server node. To not upload partially-written log files,
-the uploader will not upload files with an mtime of less than two hours ago.
-Rackspace runs this process once an hour via cron.
-
---------------
-Log Processing
---------------
-
-swift-log-stats-collector accepts a config file and generates a csv that is
-uploaded to swift. It loads all plugins defined in the config file, generates
-a list of all log files in swift that need to be processed, and passes an
-iterable of the log file data to the appropriate plugin's process method. The
-process method returns a dictionary of data in the log file keyed on (account,
-year, month, day, hour). The log-stats-collector process then combines all
-dictionaries from all calls to a process method into one dictionary. Key
-collisions within each (account, year, month, day, hour) dictionary are
-summed. Finally, the summed dictionary is mapped to the final csv values with
-each plugin's keylist_mapping method.
-
-The resulting csv file has one line per (account, year, month, day, hour) for
-all log files processed in that run of swift-log-stats-collector.
-
-
---------------------------------
-Running the stats system on SAIO
---------------------------------
-
-#. Create a swift account to use for storing stats information, and note the
-   account hash. The hash will be used in config files.
-
-#. Edit /etc/rsyslog.d/10-swift.conf::
-
-    # Uncomment the following to have a log containing all logs together
-    #local1,local2,local3,local4,local5.*   /var/log/swift/all.log
-
-    $template HourlyProxyLog,"/var/log/swift/hourly/%$YEAR%%$MONTH%%$DAY%%$HOUR%"
-    local1.*;local1.!notice ?HourlyProxyLog
-
-    local1.*;local1.!notice /var/log/swift/proxy.log
-    local1.notice           /var/log/swift/proxy.error
-    local1.*                ~
-
-#. Edit /etc/rsyslog.conf and make the following change::
-    $PrivDropToGroup adm
-
-#. `mkdir -p /var/log/swift/hourly`
-#. `chown -R syslog.adm /var/log/swift`
-#. `chmod 775 /var/log/swift /var/log/swift/hourly`
-#. `service rsyslog restart`
-#. `usermod -a -G adm <your-user-name>`
-#. Relogin to let the group change take effect.
-#. Create `/etc/swift/log-processor.conf`::
-
-    [log-processor]
-    swift_account = <your-stats-account-hash>
-    user = <your-user-name>
-
-    [log-processor-access]
-    swift_account = <your-stats-account-hash>
-    container_name = log_data
-    log_dir = /var/log/swift/hourly/
-    source_filename_pattern = ^
-        (?P<year>[0-9]{4})
-        (?P<month>[0-1][0-9])
-        (?P<day>[0-3][0-9])
-        (?P<hour>[0-2][0-9])
-        .*$
-    class_path = slogging.access_processor.AccessLogProcessor
-    user = <your-user-name>
-
-    [log-processor-stats]
-    swift_account = <your-stats-account-hash>
-    container_name = account_stats
-    log_dir = /var/log/swift/stats/
-    class_path = slogging.stats_processor.StatsLogProcessor
-    devices = /srv/1/node
-    mount_check = false
-    user = <your-user-name>
-
-    [log-processor-container-stats]
-    swift_account = <your-stats-account-hash>
-    container_name = container_stats
-    log_dir = /var/log/swift/stats/
-    class_path = slogging.stats_processor.StatsLogProcessor
-    processable = false
-    devices = /srv/1/node
-    mount_check = false
-    user = <your-user-name>
-
-#. Add the following under [app:proxy-server] in `/etc/swift/proxy-server.conf`::
-
-    log_facility = LOG_LOCAL1
-
-#. Create a `cron` job to run once per hour to create the stats logs. In
-   `/etc/cron.d/swift-stats-log-creator`::
-
-    0 * * * * <your-user-name> /usr/local/bin/swift-account-stats-logger /etc/swift/log-processor.conf
-
-#. Create a `cron` job to run once per hour to create the container stats logs. In
-   `/etc/cron.d/swift-container-stats-log-creator`::
-
-    5 * * * * <your-user-name> /usr/local/bin/swift-container-stats-logger /etc/swift/log-processor.conf
-
-#. Create a `cron` job to run once per hour to upload the stats logs. In
-   `/etc/cron.d/swift-stats-log-uploader`::
-
-    10 * * * * <your-user-name> /usr/local/bin/swift-log-uploader /etc/swift/log-processor.conf stats
-
-#. Create a `cron` job to run once per hour to upload the stats logs. In
-   `/etc/cron.d/swift-stats-log-uploader`::
-
-    15 * * * * <your-user-name> /usr/local/bin/swift-log-uploader /etc/swift/log-processor.conf container-stats
-
-#. Create a `cron` job to run once per hour to upload the access logs. In
-   `/etc/cron.d/swift-access-log-uploader`::
-
-    5 * * * * <your-user-name> /usr/local/bin/swift-log-uploader /etc/swift/log-processor.conf access
-
-#. Create a `cron` job to run once per hour to process the logs. In
-   `/etc/cron.d/swift-stats-processor`::
-
-    30 * * * * <your-user-name> /usr/local/bin/swift-log-stats-collector /etc/swift/log-processor.conf
-
-After running for a few hours, you should start to see .csv files in the
-log_processing_data container in the swift stats account that was created
-earlier. This file will have one entry per account per hour for each account
-with activity in that hour. One .csv file should be produced per hour. Note
-that the stats will be delayed by at least two hours by default. This can be
-changed with the new_log_cutoff variable in the config file. See
-`log-processor.conf-sample` for more details.

doc/source/user/how_slogging_process_logs.rst

@@ -0,0 +1,129 @@
+=================================
+How slogging process Swift's logs
+=================================
+
+This page shows you how slogging process logs on OpenStack Swift.
+
+Log Processing plugins
+~~~~~~~~~~~~~~~~~~~~~~
+
+slogging is written to allow a plugin to be defined for
+every log type.
+
+slogging includes plugins for both access logs and storage stats logs.
+
+Each plugin is responsible for defining, in a config section, where the logs
+are stored on disk, where the logs will be stored in swift (account and
+container), the filename format of the logs on disk, the location of the
+plugin class definition, and any plugin-specific config values.
+
+You need to define three methods for plugin.
+
+- The ``constructor``. must accept one argument (the dict representation of the plugin's config section).
+- The ``process`` method must accept an iterator, and the account, container, and object name of the log.
+- The ``keylist_mapping`` accepts no parameters.
+
+Actually, slogging collects following logs from Swift by using plugins.
+
+Log Uploading
+~~~~~~~~~~~~~
+
+As a first step to collect stats data from Swift, you need to use ``swift-log-uploader``.
+Basically there are three kind of logs.
+
+- :ref:`Access Logs <access-logs>`
+- :ref:`Account Logs <stats-logs>`
+- :ref:`Container DB Stats Logs <stats-logs>`
+
+You can pass plugin's name as argument of ``swift-log-uploader`` like following.::
+
+    /usr/local/bin/swift-log-uploader /etc/swift/log-processor.conf access
+    /usr/local/bin/swift-log-uploader /etc/swift/log-processor.conf stats
+    /usr/local/bin/swift-log-uploader /etc/swift/log-processor.conf container-stats
+
+You can set above command as cron job so that you can collect those kind of logs in regular basis.
+
+``swift-log-uploader`` receive a config file and a plugin name(access,
+container-stats, stats) through above settings.
+Then it finds the log files on disk according to the plugin config section and
+uploads them to the swift cluster as source data for :ref:`log-processing`.
+
+This means one uploader process will run on each proxy server node and each
+account server node.
+To not upload partially-written log files, the uploader will not upload files
+with an mtime of less than two hours ago.
+
+.. _access-logs:
+
+Access Logs
+-----------
+
+Access logs means the proxy server logs.
+
+For example, a proxy request that is made on August 4, 2010 at 12:37 gets
+logged in a file named 2010080412.
+
+This allows easy log rotation and easy per-hour log processing.
+
+To upload access logs, you can set cron like following::
+
+    /usr/local/bin/swift-log-uploader /etc/swift/log-processor.conf access
+
+
+.. _stats-logs:
+
+Account / Container DB Stats Logs
+---------------------------------
+
+You can use ``swift-account-stats-logger`` and ``swift-container-stats-logger``
+to collect Account / Container DB stats logs:
+
+``swift-account-stats-logger`` runs on each account server (via cron) and
+walks the filesystem looking for account databases. When an account database
+is found, the logger selects the account hash, bytes_used, container_count,
+and object_count. These values are then written out as one line in a csv file.
+
+One csv file is produced for every run of ``swift-account-stats-logger``.
+This means that, system wide, one csv file is produced for every storage node.
+Rackspace runs the account stats logger every hour.
+
+If you run account stats logger in every hour and if you have ten account servers,
+ten csv files are produced every hour. Also, every account will have one
+entry for every replica in the system. On average, there will be three copies
+of each account in the aggregate of all account stat csv files created in one
+system-wide run.
+
+The ``swift-container-stats-logger`` runs in a similar fashion, scanning
+the container dbs.
+
+To upload account stats logs and container stats logs, you can set cron like following::
+
+    /usr/local/bin/swift-log-uploader /etc/swift/log-processor.conf container-stats
+    /usr/local/bin/swift-log-uploader /etc/swift/log-processor.conf stats
+
+.. _log-processing:
+
+Log Processing
+~~~~~~~~~~~~~~
+
+Log Processing is a kind of final process to create total stats data.
+
+``swift-log-stats-collector`` accepts a config file and generates a csv
+that is uploaded to swift.
+
+It loads all plugins defined in the config file,
+generates a list of all log files in swift that need to be processed,
+and passes an iterable of the log file data to the appropriate plugin's
+process method.
+The process method returns a dictionary of data in the log file
+keyed on (account, year, month, day, hour).
+The ``log-stats-collector`` process then combines all dictionaries from
+all calls to a process method into one dictionary.
+Key collisions within each (account, year, month, day, hour) dictionary are
+summed.
+Finally, the summed dictionary is mapped to the final csv values with
+each plugin's ``keylist_mapping`` method.
+
+The resulting csv file has one line per (account, year, month, day, hour) for
+all log files processed in that run of ``swift-log-stats-collector``.
+

doc/source/user/index.rst

@@ -0,0 +1,8 @@
+==================
+User Documentation
+==================
+
+.. toctree::
+   :maxdepth: 1
+
+   how_slogging_process_logs

requirements.txt

@@ -0,0 +1,4 @@
+pbr!=2.1.0,>=2.0.0 # Apache-2.0
+iptools>=0.4
+tzlocal
+swift>=2.14.0

setup.cfg

@@ -1,23 +1,55 @@
+[metadata]
+name = slogging
+summary = Stats System for OpenStack Swift
+description-file =
+    README.rst
+author = OpenStack
+author-email = openstack-dev@lists.openstack.org
+home-page = https://docs.openstack.org/slogging/latest/
+classifier =
+    Environment :: OpenStack
+    Intended Audience :: Information Technology
+    Intended Audience :: System Administrators
+    License :: OSI Approved :: Apache Software License
+    Operating System :: POSIX :: Linux
+    Programming Language :: Python
+    Programming Language :: Python :: 2
+    Programming Language :: Python :: 2.7
+
+[pbr]
+skip_authors = True
+skip_changelog = True
+
+[files]
+packages =
+    slogging
+data_files =
+    etc =
+        etc/access-log-delivery.conf-sample
+        etc/log-processor.conf-sample
+scripts =
+    bin/swift-account-stats-logger
+    bin/swift-container-stats-logger
+    bin/swift-log-stats-collector
+    bin/swift-log-uploader
+    bin/swift-access-log-delivery
+
 [build_sphinx]
 all_files = 1
 build-dir = doc/build
 source-dir = doc/source
-
-[egg_info]
-tag_build = 
-tag_date = 0
-tag_svn_revision = 0
+warning-is-error = 1
 
 [compile_catalog]
 directory = locale
-domain = swauth
+domain = slogging
 
 [update_catalog]
 domain = slogging
-output_dir = locale
-input_file = locale/slogging.pot
+output_dir = slogging/locale
+input_file = slogging/locale/slogging.pot
 
 [extract_messages]
 keywords = _ l_ lazy_gettext
 mapping_file = babel.cfg
-output_file = locale/slogging.pot
+output_file = slogging/locale/slogging.pot

setup.py

@@ -14,35 +14,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from setuptools import setup, find_packages
+import setuptools
 
-from slogging import __version__ as version
+try:
+    import multiprocessing  # noqa
+except ImportError:
+    pass
 
 
-name = 'slogging'
-
-
-setup(
-    name=name,
-    version=version,
-    description='Slogging',
-    license='Apache License (2.0)',
-    author='OpenStack, LLC.',
-    author_email='me@not.mn',
-    url='https://github.com/notmyname/slogging',
-    packages=find_packages(exclude=['test_slogging', 'bin']),
-    test_suite='nose.collector',
-    classifiers=[
-        'Development Status :: 4 - Beta',
-        'License :: OSI Approved :: Apache Software License',
-        'Operating System :: POSIX :: Linux',
-        'Programming Language :: Python :: 2.6',
-        'Environment :: No Input/Output (Daemon)',
-        ],
-    install_requires=[],  # removed for better compat
-    scripts=['bin/swift-account-stats-logger',
-             'bin/swift-container-stats-logger',
-             'bin/swift-log-stats-collector', 'bin/swift-log-uploader',
-             'bin/swift-access-log-delivery'
-        ],
-    )
+setuptools.setup(
+    setup_requires=['pbr'],
+    pbr=True,
+)

slogging/__init__.py

@@ -1,8 +1,4 @@
 import gettext
 
-#: Version information (major, minor, revision[, 'dev']).
-version_info = (1, 1, 8)
-#: Version string 'major.minor.revision'.
-version = __version__ = ".".join(map(str, version_info))
 
 gettext.install('slogging')

slogging/access_log_delivery.py

@@ -13,25 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import time
 import collections
-import datetime
-from uuid import uuid4
-import Queue
-from urllib import unquote
-import os
-import cPickle
-import cStringIO
 import functools
+import os
 import random
-import errno
+import time
+
+from urllib import unquote
+from uuid import uuid4
+
+from slogging.file_buffer import FileBuffer
+from slogging import log_common
 
 from swift.common.daemon import Daemon
-from swift.common.utils import get_logger, TRUE_VALUES, split_path
-from swift.common.exceptions import LockTimeout, ChunkReadTimeout
-from slogging.log_common import LogProcessorCommon, multiprocess_collate, \
-                                   BadFileDownload
-from slogging.file_buffer import FileBuffer
+from swift.common import utils
 
 
 month_map = '_ Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split()
@@ -71,34 +66,34 @@ def memoize(func):
     return wrapped
 
 
-class AccessLogDelivery(LogProcessorCommon):
+class AccessLogDelivery(log_common.LogProcessorCommon):
 
     def __init__(self, conf, logger):
         super(AccessLogDelivery, self).__init__(conf, logger,
                                                 'access-log-delivery')
         self.frequency = int(conf.get('frequency', '3600'))
-        self.metadata_key = conf.get('metadata_key',
-                                'x-container-meta-access-log-delivery').lower()
+        self.metadata_key = conf.get(
+            'metadata_key',
+            'x-container-meta-access-log-delivery').lower()
         self.server_name = conf.get('server_name', 'proxy-server')
         self.working_dir = conf.get('working_dir', '/tmp/swift').rstrip('/')
         buffer_limit = conf.get('buffer_limit', '10485760')
         self.file_buffer = FileBuffer(buffer_limit, logger)
         self.hidden_ips = [x.strip() for x in
-                            conf.get('hidden_ips', '').split(',') if x.strip()]
+                           conf.get('hidden_ips', '').split(',') if x.strip()]
         self.source_account = conf['log_source_account']
         self.source_container = conf.get('log_source_container_name',
                                          'log_data')
 
     def get_logs_to_process(self, already_processed_files):
         lookback_start, lookback_end = self.calculate_lookback()
-        logs_to_process = self.get_container_listing(
-                                                self.source_account,
-                                                self.source_container,
-                                                lookback_start,
-                                                lookback_end,
-                                                already_processed_files)
+        logs_to_process = self.get_container_listing(self.source_account,
+                                                     self.source_container,
+                                                     lookback_start,
+                                                     lookback_end,
+                                                     already_processed_files)
         logs_to_process = [(self.source_account, self.source_container, x)
-                            for x in logs_to_process]
+                           for x in logs_to_process]
         self.logger.info(_('loaded %d files to process') %
                          len(logs_to_process))
         return logs_to_process
@@ -108,16 +103,16 @@ class AccessLogDelivery(LogProcessorCommon):
         try:
             year, month, day, hour, _unused = object_name.split('/', 4)
         except ValueError:
-            self.logger.info(_('Odd object name: %s. Skipping' % object_name))
+            self.logger.info(_('Odd object name: %s. Skipping') % object_name)
             return
         filename_pattern = '%s/%%s/%%s/%s/%s/%s/%s' % (self.working_dir, year,
-                                                   month, day, hour)
-        self.logger.debug(_('Processing %s' % object_name))
+                                                       month, day, hour)
+        self.logger.debug(_('Processing %s') % object_name)
         # get an iter of the object data
         compressed = object_name.endswith('.gz')
         stream = self.get_object_data(account, container, object_name,
                                       compressed=compressed)
-        buff = collections.defaultdict(list)
+        collections.defaultdict(list)
         for line in stream:
             clf, account, container = self.convert_log_line(line)
             if not clf or not account or not container:
@@ -138,7 +133,7 @@ class AccessLogDelivery(LogProcessorCommon):
             metadata = self.internal_proxy.get_container_metadata(account,
                                                                   container)
             val = metadata.get(self.metadata_key, '')
-            flag = val.lower() in TRUE_VALUES
+            flag = val.lower() in utils.TRUE_VALUES
             self.memcache.set(key, flag, timeout=self.frequency)
         return flag
 
@@ -161,38 +156,39 @@ class AccessLogDelivery(LogProcessorCommon):
                     # internal proxy log
                     return {}
             (unused,
-            server,
-            client_ip,
-            lb_ip,
-            timestamp,
-            method,
-            request,
-            http_version,
-            code,
-            referrer,
-            user_agent,
-            auth_token,
-            bytes_in,
-            bytes_out,
-            etag,
-            trans_id,
-            headers,
-            processing_time) = (unquote(x) for x in log_arr[:18])
+             server,
+             client_ip,
+             lb_ip,
+             timestamp,
+             method,
+             request,
+             http_version,
+             code,
+             referrer,
+             user_agent,
+             auth_token,
+             bytes_in,
+             bytes_out,
+             etag,
+             trans_id,
+             headers,
+             processing_time) = (unquote(x) for x in log_arr[:18])
         except ValueError:
             self.logger.debug(_('Bad line data: %s') % repr(raw_log))
             return {}
         if server != self.server_name:
             # incorrect server name in log line
-            self.logger.debug(_('Bad server name: found "%(found)s" ' \
-                    'expected "%(expected)s"') %
-                    {'found': server, 'expected': self.server_name})
+            self.logger.debug(
+                _('Bad server name: found "%(found)s" '
+                  'expected "%(expected)s"') %
+                {'found': server, 'expected': self.server_name})
             return {}
         try:
             (version, account, container_name, object_name) = \
-                split_path(request, 2, 4, True)
-        except ValueError, e:
+                utils.split_path(request, 2, 4, True)
+        except ValueError as e:
             self.logger.debug(_('Invalid path: %(error)s from data: %(log)s') %
-            {'error': e, 'log': repr(raw_log)})
+                              {'error': e, 'log': repr(raw_log)})
             return {}
         if container_name is not None:
             container_name = container_name.split('?', 1)[0]
@@ -234,15 +230,16 @@ class AccessLogDelivery(LogProcessorCommon):
 
 
 class AccessLogDeliveryDaemon(Daemon):
-    """
-    Processes access (proxy) logs to split them up by account and deliver the
-    split logs to their respective accounts.
+    """AccessLogDeliveryDaemon class.
+
+    Processes access (proxy) logs to split them up by
+    account and deliver the split logs to their respective accounts.
     """
 
     def __init__(self, c):
         self.conf = c
         super(AccessLogDeliveryDaemon, self).__init__(c)
-        self.logger = get_logger(c, log_route='access-log-delivery')
+        self.logger = utils.get_logger(c, log_route='access-log-delivery')
         self.log_processor = AccessLogDelivery(c, self.logger)
         self.log_delivery_account = c['swift_account']
         self.log_delivery_container = c.get('container_name',
@@ -269,16 +266,19 @@ class AccessLogDeliveryDaemon(Daemon):
             self.log_processor.get_logs_to_process(already_processed_files)
         if not logs_to_process:
             self.logger.info(_("Log processing done (%0.2f minutes)") %
-                        ((time.time() - start) / 60))
+                             ((time.time() - start) / 60))
             return
 
         # map
         processor_args = (self.conf, self.logger)
-        results = multiprocess_collate(AccessLogDelivery, processor_args,
-                                       'process_one_file', logs_to_process,
-                                       self.worker_count)
+        results = log_common.multiprocess_collate(
+            AccessLogDelivery,
+            processor_args,
+            'process_one_file',
+            logs_to_process,
+            self.worker_count)
 
-        #reduce
+        # reduce
         processed_files = already_processed_files
         files_to_upload = set()
         for item, data in results:
@@ -292,24 +292,25 @@ class AccessLogDeliveryDaemon(Daemon):
             account, target_name = target_name.split('/', 1)
             some_id = uuid4().hex
             target_name = '%s/%s.log.gz' % (target_name, some_id)
-            success = self.log_processor.internal_proxy.upload_file(filename,
-                            account,
-                            self.target_container,
-                            target_name)
+            success = self.log_processor.internal_proxy.upload_file(
+                filename,
+                account,
+                self.target_container,
+                target_name)
             if success:
                 os.unlink(filename)
                 self.logger.debug('Uploaded %s to account %s' % (filename,
                                                                  account))
             else:
-                self.logger.error('Could not upload %s to account %s' % (
-                                    filename, account))
+                self.logger.error('Could not upload %s to account %s' %
+                                  (filename, account))
 
         # cleanup
         success = self.log_processor.save_processed_files(processed_files)
         if not success:
             self.logger.error('Error uploading updated processed files log')
         self.logger.info(_("Log processing done (%0.2f minutes)") %
-                    ((time.time() - start) / 60))
+                         ((time.time() - start) / 60))
 
     def run_forever(self, *a, **kw):
         while True:

slogging/access_processor.py

@@ -13,13 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import collections
-from urllib import unquote
-import copy
-from tzlocal import get_localzone
 from datetime import datetime
-from slogging import common
 import pytz
+from slogging import common
+from tzlocal import get_localzone
+from urllib import unquote
 from urlparse import urlparse
 
 # conditionalize the return_ips method based on whether or not iptools
@@ -29,9 +27,10 @@ try:
     CIDR_support = True
 
     def return_ips(conf, conf_tag):
-        return set(k for k in
-                    IpRangeList(*[x.strip() for x in
-                    conf.get(conf_tag, '').split(',') if x.strip()]))
+        return set(k for k in IpRangeList(*[
+            x.strip() for x in conf.get(conf_tag, '').split(',')
+            if x.strip()]))
+
     def sanitize_ips(line_data):
         for x in ['lb_ip', 'client_ip', 'log_source']:
             if line_data[x] == '-':
@@ -40,26 +39,29 @@ except ImportError:
     CIDR_support = False
 
     def return_ips(conf, conf_tag):
-        return ([x.strip() for x in conf.get(conf_tag, '').split(',')
+        return ([
+            x.strip() for x in conf.get(conf_tag, '').split(',')
             if x.strip()])
 
-from swift.common.utils import split_path, get_logger
+from swift.common import utils
 
 month_map = '_ Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split()
 LISTING_PARAMS = set(
-                'path limit format delimiter marker end_marker prefix'.split())
+    'path limit format delimiter marker end_marker prefix'.split())
 local_zone = get_localzone()
 
 
 class AccessLogProcessor(object):
-    """Transform proxy server access logs"""
+    """AccessLogProcessor class.
 
+    Transform proxy server access logs
+    """
     def __init__(self, conf):
         self.server_name = conf.get('server_name', 'proxy-server')
         for conf_tag in ['lb_private_ips', 'service_ips']:
             setattr(self, conf_tag, return_ips(conf, conf_tag))
         self.warn_percent = float(conf.get('warn_percent', '0.8'))
-        self.logger = get_logger(conf, log_route='access-processor')
+        self.logger = utils.get_logger(conf, log_route='access-processor')
         self.time_zone = common.get_time_zone(conf, self.logger, 'time_zone',
                                               str(local_zone))
 
@@ -70,23 +72,23 @@ class AccessLogProcessor(object):
             log_source = None
             split_log = raw_log[16:].split(' ')
             (unused,
-            server,
-            client_ip,
-            lb_ip,
-            timestamp,
-            method,
-            request,
-            http_version,
-            code,
-            referrer,
-            user_agent,
-            auth_token,
-            bytes_in,
-            bytes_out,
-            etag,
-            trans_id,
-            headers,
-            processing_time) = (unquote(x) for x in split_log[:18])
+             server,
+             client_ip,
+             lb_ip,
+             timestamp,
+             method,
+             request,
+             http_version,
+             code,
+             referrer,
+             user_agent,
+             auth_token,
+             bytes_in,
+             bytes_out,
+             etag,
+             trans_id,
+             headers,
+             processing_time) = (unquote(x) for x in split_log[:18])
             if len(split_log) > 18:
                 log_source = split_log[18]
         except ValueError:
@@ -94,26 +96,27 @@ class AccessLogProcessor(object):
             return {}
         if server != self.server_name:
             # incorrect server name in log line
-            self.logger.debug(_('Bad server name: found "%(found)s" ' \
-                    'expected "%(expected)s"') %
-                    {'found': server, 'expected': self.server_name})
+            self.logger.debug(_('Bad server name: found "%(found)s" '
+                                'expected "%(expected)s"') %
+                              {'found': server, 'expected': self.server_name})
             return {}
         try:
             parsed_url = urlparse(request)
             request = parsed_url.path
             query = parsed_url.query
             (version, account, container_name, object_name) = \
-                split_path(request, 2, 4, True)
-        except ValueError, e:
-            self.logger.debug(_('Invalid path: %(error)s from data: %(log)s') %
-            {'error': e, 'log': repr(raw_log)})
+                utils.split_path(request, 2, 4, True)
+        except ValueError as e:
+            self.logger.debug(
+                _('Invalid path: %(error)s from data: %(log)s') %
+                {'error': e, 'log': repr(raw_log)})
             return {}
         if version != 'v1':
             # "In the wild" places this can be caught are with auth systems
             # that use the same endpoint as the rest of the Swift API (eg
             # tempauth or swauth). But if the Swift API ever does change, this
             # protects that too.
-            self.logger.debug(_('Unexpected Swift version string: found ' \
+            self.logger.debug(_('Unexpected Swift version string: found '
                                 '"%s" expected "v1"') % version)
             return {}
         if query != "":
@@ -187,7 +190,7 @@ class AccessLogProcessor(object):
             method = line_data['method']
             code = int(line_data['code'])
             object_name = line_data['object_name']
-            client_ip = line_data['client_ip']
+            # client_ip = line_data['client_ip']
 
             op_level = None
             if not container_name:
@@ -219,17 +222,18 @@ class AccessLogProcessor(object):
 
             d[(source, 'bytes_out')] = d.setdefault((
                 source, 'bytes_out'), 0) + bytes_out
-            d[(source, 'bytes_in')] = d.setdefault((source, 'bytes_in'), 0) + \
-                                      bytes_in
+            d[(source, 'bytes_in')] = \
+                d.setdefault((source, 'bytes_in'), 0) + bytes_in
 
-            d['format_query'] = d.setdefault('format_query', 0) + \
-                                line_data.get('format', 0)
-            d['marker_query'] = d.setdefault('marker_query', 0) + \
-                                line_data.get('marker', 0)
-            d['prefix_query'] = d.setdefault('prefix_query', 0) + \
-                                line_data.get('prefix', 0)
-            d['delimiter_query'] = d.setdefault('delimiter_query', 0) + \
-                                   line_data.get('delimiter', 0)
+            d['format_query'] = \
+                d.setdefault('format_query', 0) + line_data.get('format', 0)
+            d['marker_query'] = \
+                d.setdefault('marker_query', 0) + line_data.get('marker', 0)
+            d['prefix_query'] = \
+                d.setdefault('prefix_query', 0) + line_data.get('prefix', 0)
+            d['delimiter_query'] = \
+                d.setdefault('delimiter_query', 0) + line_data.get('delimiter',
+                                                                   0)
             path = line_data.get('path', 0)
             d['path_query'] = d.setdefault('path_query', 0) + path
 
@@ -241,9 +245,12 @@ class AccessLogProcessor(object):
         if bad_lines > (total_lines * self.warn_percent):
             name = '/'.join([data_object_account, data_object_container,
                              data_object_name])
-            self.logger.warning(_('I found a bunch of bad lines in %(name)s '\
-                    '(%(bad)d bad, %(total)d total)') %
-                    {'name': name, 'bad': bad_lines, 'total': total_lines})
+            self.logger.warning(_(
+                'I found a bunch of bad lines in %(name)s '
+                '(%(bad)d bad, %(total)d total)') % {
+                'name': name,
+                'bad': bad_lines,
+                'total': total_lines})
         return hourly_aggr_info
 
     def keylist_mapping(self):
@@ -253,7 +260,7 @@ class AccessLogProcessor(object):
         code_keys = '2xx 4xx 5xx'.split()
 
         keylist_mapping = {
-        #   <db key> : <row key> or <set of row keys>
+            #   <db key> : <row key> or <set of row keys>
             'service_bw_in': ('service', 'bytes_in'),
             'service_bw_out': ('service', 'bytes_out'),
             'public_bw_in': ('public', 'bytes_in'),
@@ -274,19 +281,19 @@ class AccessLogProcessor(object):
                 for verb in verb_keys:
                     for code in code_keys:
                         keylist_mapping['account_requests'].add(
-                                        (source, 'account', verb, code))
+                            (source, 'account', verb, code))
                         keylist_mapping['container_requests'].add(
-                                        (source, 'container', verb, code))
+                            (source, 'container', verb, code))
                         keylist_mapping['object_requests'].add(
-                                        (source, 'object', verb, code))
+                            (source, 'object', verb, code))
                         keylist_mapping['service_request'].add(
-                                        ('service', level, verb, code))
+                            ('service', level, verb, code))
                         keylist_mapping['public_request'].add(
-                                        ('public', level, verb, code))
+                            ('public', level, verb, code))
                         keylist_mapping[verb].add(
-                                        (source, level, verb, code))
+                            (source, level, verb, code))
                         keylist_mapping[code].add(
-                                        (source, level, verb, code))
+                            (source, level, verb, code))
                         keylist_mapping['ops_count'].add(
-                                        (source, level, verb, code))
+                            (source, level, verb, code))
         return keylist_mapping

slogging/common.py

@@ -17,28 +17,30 @@ import pytz
 
 
 def get_time_zone(conf, logger, key, default):
-    """
-    Get and check time_zone value.
-    """
+    """Get and check time_zone value."""
     str_time_zone = conf.get(key, default)
     try:
         time_zone = pytz.timezone(str_time_zone)
     except pytz.exceptions.UnknownTimeZoneError:
-        logger.warning(
-            _("Invalid Parameter %s: %s, " % (key, str_time_zone) +
-              "use default %s.") % default)
+        msg = _("Invalid Parameter %(key)s: %(str_time_zone)s, "
+                "use default %(default)s.") % {'key': key,
+                                               'str_time_zone': str_time_zone,
+                                               'default': default}
+        logger.warning(msg)
         time_zone = pytz.timezone(default)
     return time_zone
 
 
 def get_format_type(conf, logger, key, default):
-    """
-    Get and check format_type value.
-    """
+    """Get and check format_type value."""
     format_type = conf.get(key, default).lower()
     if format_type not in ('json', 'csv'):
-        logger.warning(
-            _("Invalid Parameter %s: %s, " % (key, format_type) +
-              "use default %s.") % default)
+        # msg = _("Invalid Parameter %s: %s, " % (key, format_type) +
+        #         "use default %s.") % default
+        msg = _("Invalid Parameter %(key)s: %(format_type)s, "
+                "use default %(default)s.") % {'key': key,
+                                               'format_type': format_type,
+                                               'default': default}
+        logger.warning(msg)
         format_type = default
     return format_type

slogging/compressing_file_reader.py

@@ -13,12 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import zlib
 import struct
+import zlib
 
 
 class CompressingFileReader(object):
-    '''
+    """CompressingFileReader class.
+
     Wraps a file object and provides a read method that returns gzip'd data.
 
     One warning: if read is called with a small value, the data returned may
@@ -35,8 +36,7 @@ class CompressingFileReader(object):
 
     :param file_obj: File object to read from
     :param compresslevel: compression level
-    '''
-
+    """
     def __init__(self, file_obj, compresslevel=9):
         self._f = file_obj
         self._compressor = zlib.compressobj(compresslevel,

slogging/db_stats_collector.py

@@ -13,37 +13,37 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-import time
 from datetime import datetime
-from paste.deploy import appconfig
-import shutil
 import hashlib
-import urllib
+import os
+import shutil
+from slogging import common
 import sqlite3
 
 from swift.account.backend import AccountBroker
 from swift.account.server import DATADIR as account_server_data_dir
-from swift.container.backend import ContainerBroker
-from swift.container.server import DATADIR as container_server_data_dir
-from swift.common.utils import renamer, get_logger, readconf, mkdirs, \
-    TRUE_VALUES, remove_file
 from swift.common.constraints import check_mount
 from swift.common.daemon import Daemon
-from slogging import common
+from swift.common import utils
+from swift.container.backend import ContainerBroker
+from swift.container.server import DATADIR as container_server_data_dir
+
+import time
 from tzlocal import get_localzone
+import urllib
+
 
 local_zone = get_localzone()
 
 
 class DatabaseStatsCollector(Daemon):
-    """
-    Extract storage stats from account databases on the account
-    storage nodes
+    """DatabaseStatsCollector class.
+
+    Extract storage stats from account databases on the
+    account storage nodes.
 
     Any subclasses must define the function get_data.
     """
-
     def __init__(self, stats_conf, stats_type, data_dir, filename_format):
         super(DatabaseStatsCollector, self).__init__(stats_conf)
         self.stats_type = stats_type
@@ -51,20 +51,23 @@ class DatabaseStatsCollector(Daemon):
         self.filename_format = filename_format
         self.devices = stats_conf.get('devices', '/srv/node')
         self.mount_check = stats_conf.get('mount_check',
-                                           'true').lower() in TRUE_VALUES
+                                          'true').lower() in utils.TRUE_VALUES
         self.target_dir = stats_conf.get('log_dir', '/var/log/swift')
-        mkdirs(self.target_dir)
-        self.logger = get_logger(stats_conf,
-                                 log_route='%s-stats' % stats_type)
+        utils.mkdirs(self.target_dir)
+        self.logger = utils.get_logger(stats_conf,
+                                       log_route='%s-stats' % stats_type)
         self.time_zone = common.get_time_zone(stats_conf, self.logger,
                                               'time_zone', str(local_zone))
 
     def run_once(self, *args, **kwargs):
-        self.logger.info(_("Gathering %s stats" % self.stats_type))
+        self.logger.info(_("Gathering %s stats") % self.stats_type)
         start = time.time()
         self.find_and_process()
-        self.logger.info(_("Gathering %s stats complete (%0.2f minutes)") %
-                         (self.stats_type, (time.time() - start) / 60))
+
+        msg = _("Gathering %(stats_type)s stats complete "
+                "(%(time)0.2f minutes)") % {'stats_type': self.stats_type,
+                                            'time': (time.time() - start) / 60}
+        self.logger.info(msg)
 
     def get_data(self):
         raise NotImplementedError('Subclasses must override')
@@ -78,7 +81,7 @@ class DatabaseStatsCollector(Daemon):
         working_dir = os.path.join(self.target_dir,
                                    '.%-stats_tmp' % self.stats_type)
         shutil.rmtree(working_dir, ignore_errors=True)
-        mkdirs(working_dir)
+        utils.mkdirs(working_dir)
         tmp_filename = os.path.join(working_dir, src_filename)
         hasher = hashlib.md5()
         try:
@@ -101,35 +104,37 @@ class DatabaseStatsCollector(Daemon):
                                 db_path = os.path.join(root, filename)
                                 try:
                                     line_data = self.get_data(db_path)
-                                except sqlite3.Error, err:
-                                    self.logger.info(
-                                        _("Error accessing db %s: %s") %
-                                          (db_path, err))
+                                except sqlite3.Error as err:
+                                    values = {'db_path': db_path, 'err': err}
+                                    msg = _("Error accessing db "
+                                            "%(db_path)s: %(err)s") % values
+                                    self.logger.info(msg)
                                     continue
                                 if line_data:
                                     statfile.write(line_data)
                                     hasher.update(line_data)
 
             src_filename += hasher.hexdigest()
-            renamer(tmp_filename, os.path.join(self.target_dir, src_filename))
+            utils.renamer(tmp_filename,
+                          os.path.join(self.target_dir, src_filename))
         finally:
             shutil.rmtree(working_dir, ignore_errors=True)
 
 
 class AccountStatsCollector(DatabaseStatsCollector):
-    """
+    """AccountStatsCollector class.
+
     Extract storage stats from account databases on the account
     storage nodes
     """
-
     def __init__(self, stats_conf):
         super(AccountStatsCollector, self).__init__(stats_conf, 'account',
                                                     account_server_data_dir,
                                                     'stats-%Y%m%d%H_')
 
     def get_data(self, db_path):
-        """
-        Data for generated csv has the following columns:
+        """Data for generated csv has the following columns:
+
         Account Hash, Container Count, Object Count, Bytes Used
 
         :raises sqlite3.Error: does not catch errors connecting to db
@@ -149,19 +154,21 @@ class AccountStatsCollector(DatabaseStatsCollector):
 
 
 class ContainerStatsCollector(DatabaseStatsCollector):
-    """
+    """ContainerStatsCollector class
+
     Extract storage stats from container databases on the container
     storage nodes
     """
-
     def __init__(self, stats_conf):
-        super(ContainerStatsCollector, self).__init__(stats_conf, 'container',
-                                                   container_server_data_dir,
-                                                   'container-stats-%Y%m%d%H_')
+        super(ContainerStatsCollector, self).__init__(
+            stats_conf, 'container',
+            container_server_data_dir,
+            'container-stats-%Y%m%d%H_')
         # webob calls title on all the header keys
-        self.metadata_keys = ['X-Container-Meta-%s' % mkey.strip().title()
-                for mkey in stats_conf.get('metadata_keys', '').split(',')
-                if mkey.strip()]
+        self.metadata_keys = [
+            'X-Container-Meta-%s' % mkey.strip().title()
+            for mkey in stats_conf.get('metadata_keys', '').split(',')
+            if mkey.strip()]
 
     def get_header(self):
         header = 'Account Hash,Container Name,Object Count,Bytes Used'
@@ -172,9 +179,10 @@ class ContainerStatsCollector(DatabaseStatsCollector):
         return header
 
     def get_data(self, db_path):
-        """
-        Data for generated csv has the following columns:
+        """Data for generated csv has the following columns:
+
         Account Hash, Container Name, Object Count, Bytes Used
+
         This will just collect whether or not the metadata is set
         using a 1 or ''.
 

slogging/file_buffer.py

@@ -1,10 +1,27 @@
-import os
+# Copyright (c) 2010-2011 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import collections
 import errno
+import os
+from swift.common.exceptions import LockTimeout
 from swift.common.utils import lock_file
 
 
 class FileBuffer(object):
+    """FileBuffer class"""
 
     def __init__(self, limit, logger):
         self.buffers = collections.defaultdict(list)
@@ -26,7 +43,7 @@ class FileBuffer(object):
                 mid_dirs = os.path.dirname(filename)
                 try:
                     os.makedirs(mid_dirs)
-                except OSError, err:
+                except OSError as err:
                     if err.errno == errno.EEXIST:
                         pass
                     else:
@@ -36,7 +53,7 @@ class FileBuffer(object):
                         f.write(out)
                 except LockTimeout:
                     # couldn't write, we'll try again later
-                    self.logger.debug(_('Timeout writing to %s' % filename))
+                    self.logger.debug(_('Timeout writing to %s') % filename)
                 else:
                     del self.buffers[filename]
         self.total_size = 0

slogging/internal_proxy.py

@@ -13,13 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from urllib import quote, unquote
 from json import loads as json_loads
-import copy
-
 from slogging.compressing_file_reader import CompressingFileReader
-from swift.proxy.server import Application
 from swift.common import swob
+from swift.proxy.server import Application
+from urllib import quote
 
 
 def make_request_body_file(source_file, compress=True):
@@ -43,7 +41,8 @@ def swob_request_copy(orig_req, source_file=None, compress=True):
 
 
 class InternalProxy(object):
-    """
+    """Internal Proxy class.
+
     Set up a private instance of a proxy server that allows normal requests
     to be made without having to actually send the request to the proxy.
     This also doesn't log the requests to the normal proxy logs.
@@ -52,11 +51,11 @@ class InternalProxy(object):
     :param logger: logger to log requests to
     :param retries: number of times to retry each request
     """
-
     def __init__(self, proxy_server_conf=None, logger=None, retries=0,
                  memcache=None):
-        self.upload_app = Application(proxy_server_conf, memcache=memcache,
-                                          logger=logger)
+        self.upload_app = Application(proxy_server_conf,
+                                      memcache=memcache,
+                                      logger=logger)
         self.retries = retries
 
     def _handle_request(self, req, source_file=None, compress=True):
@@ -77,8 +76,7 @@ class InternalProxy(object):
     def upload_file(self, source_file, account, container, object_name,
                     compress=True, content_type='application/x-gzip',
                     etag=None, headers=None):
-        """
-        Upload a file to cloud files.
+        """Upload a file to cloud files.
 
         :param source_file: path to or file like object to upload
         :param account: account to upload to
@@ -101,8 +99,8 @@ class InternalProxy(object):
 
         # upload the file to the account
         req = swob.Request.blank(target_name,
-                            environ={'REQUEST_METHOD': 'PUT'},
-                            headers=send_headers)
+                                 environ={'REQUEST_METHOD': 'PUT'},
+                                 headers=send_headers)
         req.environ['content_type'] = content_type
         req.content_length = None   # to make sure we send chunked data
         if etag:
@@ -114,8 +112,7 @@ class InternalProxy(object):
         return True
 
     def get_object(self, account, container, object_name):
-        """
-        Get object.
+        """Get object.
 
         :param account: account name object is in
         :param container: container name object is in
@@ -123,29 +120,27 @@ class InternalProxy(object):
         :returns: iterator for object data
         """
         req = swob.Request.blank('/v1/%s/%s/%s' %
-                            (account, container, object_name),
-                            environ={'REQUEST_METHOD': 'GET'})
+                                 (account, container, object_name),
+                                 environ={'REQUEST_METHOD': 'GET'})
         resp = self._handle_request(req)
         return resp.status_int, resp.app_iter
 
     def create_container(self, account, container):
-        """
-        Create container.
+        """Create container.
 
         :param account: account name to put the container in
         :param container: container name to create
         :returns: True if successful, otherwise False
         """
         req = swob.Request.blank('/v1/%s/%s' % (account, container),
-                            environ={'REQUEST_METHOD': 'PUT'})
+                                 environ={'REQUEST_METHOD': 'PUT'})
         resp = self._handle_request(req)
         return 200 <= resp.status_int < 300
 
     def get_container_list(self, account, container, marker=None,
                            end_marker=None, limit=None, prefix=None,
                            delimiter=None, full_listing=True):
-        """
-        Get a listing of objects for the container.
+        """Get a listing of objects for the container.
 
         :param account: account name for the container
         :param container: container name to get a listing for
@@ -156,6 +151,7 @@ class InternalProxy(object):
         :param delimeter: string to delimit the queries on
         :param full_listing: if True, return a full listing, else returns a max
                              of 10000 listings
+
         :returns: list of objects
         """
         if full_listing:
@@ -190,7 +186,7 @@ class InternalProxy(object):
         req = swob.Request.blank(path, environ={'REQUEST_METHOD': 'GET'})
         resp = self._handle_request(req)
         if resp.status_int < 200 or resp.status_int >= 300:
-            return []  # TODO: distinguish between 404 and empty container
+            return []
         if resp.status_int == 204:
             return []
         return json_loads(resp.body)
@@ -200,7 +196,7 @@ class InternalProxy(object):
         req = swob.Request.blank(path, environ={'REQUEST_METHOD': 'HEAD'})
         resp = self._handle_request(req)
         out = {}
-        for k, v in resp.headers.iteritems():
+        for k, v in resp.headers.items():
             if k.lower().startswith('x-container-meta-'):
                 out[k] = v
         return out

slogging/log_common.py

@@ -13,27 +13,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import multiprocessing
-import Queue
-from datetime import datetime, timedelta
-import zlib
-import time
-from paste.deploy import appconfig
-from contextlib import contextmanager
-import os
-import errno
-import fcntl
-import sys
-import traceback
 import cPickle
 import cStringIO
-
+from datetime import datetime
+from datetime import timedelta
 from eventlet import sleep
-
-from swift.common.memcached import MemcacheRing
+import multiprocessing
+from paste.deploy import appconfig
+import Queue
 from slogging.internal_proxy import InternalProxy
+from swift.common.exceptions import ChunkReadTimeout
+from swift.common.memcached import MemcacheRing
 from swift.common.utils import get_logger
-from swift.common.exceptions import ChunkReadTimeout, LockTimeout
+import sys
+import time
+import traceback
+import zlib
 
 
 class BadFileDownload(Exception):
@@ -54,8 +49,8 @@ class LogProcessorCommon(object):
             self.logger = get_logger(*logger, log_route=log_route)
         else:
             self.logger = logger
-        self.memcache = MemcacheRing([s.strip() for s in
-            conf.get('memcache_servers', '').split(',')
+        self.memcache = MemcacheRing([
+            s.strip() for s in conf.get('memcache_servers', '').split(',')
             if s.strip()])
         self.conf = conf
         self._internal_proxy = None
@@ -64,10 +59,10 @@ class LogProcessorCommon(object):
                                    self.lookback_hours))
         self.log_processor_account = conf['swift_account']
         self.log_processor_container = conf.get('container_name',
-                                            'simple_billing_data')
+                                                'simple_billing_data')
         self.processed_files_object_name = \
-                conf.get('processed_files_object_name',
-                        'processed_files.pickle.gz')
+            conf.get('processed_files_object_name',
+                     'processed_files.pickle.gz')
 
     @property
     def internal_proxy(self):
@@ -77,12 +72,12 @@ class LogProcessorCommon(object):
             if proxy_server_conf_loc is None:
                 # then look in a section called log-processor
                 stats_conf = self.conf.get('log-processor', {})
-                proxy_server_conf_loc = stats_conf.get('proxy_server_conf',
-                                                '/etc/swift/proxy-server.conf')
+                proxy_server_conf_loc = stats_conf.get(
+                    'proxy_server_conf', '/etc/swift/proxy-server.conf')
             if proxy_server_conf_loc:
                 proxy_server_conf = appconfig(
-                                        'config:%s' % proxy_server_conf_loc,
-                                        name='proxy-server')
+                    'config:%s' % proxy_server_conf_loc,
+                    name='proxy-server')
             else:
                 proxy_server_conf = None
             self._internal_proxy = InternalProxy(proxy_server_conf,
@@ -118,47 +113,46 @@ class LogProcessorCommon(object):
             # There is not a good way to determine when an old entry should be
             # pruned (lookback_hours could be set to anything and could change)
             processed_files_stream = self.get_object_data(
-                                        self.log_processor_account,
-                                        self.log_processor_container,
-                                        self.processed_files_object_name,
-                                        compressed=True)
+                self.log_processor_account,
+                self.log_processor_container,
+                self.processed_files_object_name,
+                compressed=True)
             buf = '\n'.join(x for x in processed_files_stream)
             if buf:
                 already_processed_files = cPickle.loads(buf)
             else:
                 already_processed_files = set()
-        except BadFileDownload, err:
+        except BadFileDownload as err:
             if err.status_code == 404:
                 already_processed_files = set()
             else:
                 self.logger.error(_('Simple billing unable to load list '
-                    'of already processed log files'))
+                                    'of already processed log files'))
                 return
-        self.logger.debug(_('found %d processed files') % \
+        self.logger.debug(_('found %d processed files') %
                           len(already_processed_files))
         return already_processed_files
 
     def save_processed_files(self, processed_files_data):
         s = cPickle.dumps(processed_files_data, cPickle.HIGHEST_PROTOCOL)
         f = cStringIO.StringIO(s)
-        return self.internal_proxy.upload_file(f, self.log_processor_account,
-                                            self.log_processor_container,
-                                            self.processed_files_object_name)
+        return self.internal_proxy.upload_file(
+            f,
+            self.log_processor_account,
+            self.log_processor_container,
+            self.processed_files_object_name)
 
     def get_object_data(self, swift_account, container_name, object_name,
                         compressed=False):
         '''reads an object and yields its lines'''
-        self.logger.debug('get_object_data(%r, %r, %r, compressed=%r)' %
-                                                        (swift_account,
-                                                        container_name,
-                                                        object_name,
-                                                        compressed))
+        self.logger.debug('get_object_data(%r, %r, %r, compressed=%r)' % (
+            swift_account, container_name, object_name, compressed))
         code, o = self.internal_proxy.get_object(swift_account, container_name,
                                                  object_name)
         if code < 200 or code >= 300:
             raise BadFileDownload(code)
         last_part = ''
-        last_compressed_part = ''
+
         # magic in the following zlib.decompressobj argument is courtesy of
         # Python decompressing gzip chunk-by-chunk
         # http://stackoverflow.com/questions/2423866
@@ -169,9 +163,10 @@ class LogProcessorCommon(object):
                     try:
                         chunk = d.decompress(chunk)
                     except zlib.error:
-                        self.logger.debug(_('Bad compressed data for %s')
-                            % '/'.join((swift_account, container_name,
-                                        object_name)))
+                        self.logger.debug(_('Bad compressed data for %s') %
+                                          '/'.join((swift_account,
+                                                    container_name,
+                                                    object_name)))
                         raise BadFileDownload()  # bad compressed data
                 parts = chunk.split('\n')
                 parts[0] = last_part + parts[0]
@@ -186,10 +181,13 @@ class LogProcessorCommon(object):
     def get_container_listing(self, swift_account, container_name,
                               start_date=None, end_date=None,
                               listing_filter=None):
-        '''
-        Get a container listing, filtered by start_date, end_date, and
-        listing_filter. Dates, if given, must be in YYYYMMDDHH format
-        '''
+        """Get a container listing.
+
+        Data can be filtered by start_date, end_date, and
+        listing_filter.
+
+        Dates, if given, must be in YYYYMMDDHH format.
+        """
         search_key = None
         if start_date is not None:
             try:
@@ -218,10 +216,10 @@ class LogProcessorCommon(object):
                 hour = '%02d' % (parsed_date.tm_hour + 1)
                 end_key = '/'.join([year, month, day, hour])
         container_listing = self.internal_proxy.get_container_list(
-                                    swift_account,
-                                    container_name,
-                                    marker=search_key,
-                                    end_marker=end_key)
+            swift_account,
+            container_name,
+            marker=search_key,
+            end_marker=end_key)
         results = []
         if listing_filter is None:
             listing_filter = set()
@@ -234,8 +232,6 @@ class LogProcessorCommon(object):
 
 def multiprocess_collate(processor_klass, processor_args, processor_method,
                          items_to_process, worker_count, logger=None):
-    '''
-    '''
     results = []
     in_queue = multiprocessing.Queue()
     out_queue = multiprocessing.Queue()
@@ -275,7 +271,7 @@ def multiprocess_collate(processor_klass, processor_args, processor_method,
 
 def collate_worker(processor_klass, processor_args, processor_method, in_queue,
                    out_queue, logger=None):
-    '''worker process for multiprocess_collate'''
+    """worker process for multiprocess_collate"""
     try:
         p = processor_klass(*processor_args)
         while True:
@@ -289,7 +285,7 @@ def collate_worker(processor_klass, processor_args, processor_method, in_queue,
                 return
             try:
                 ret = method(*item)
-            except:
+            except Exception:
                 err_type, err, tb = sys.exc_info()
                 # Use err_type since unplickling err in the parent process
                 # will fail if it has a custom constructor with required
@@ -297,7 +293,7 @@ def collate_worker(processor_klass, processor_args, processor_method, in_queue,
                 ret = WorkerError()
                 ret.tb_str = ''.join(traceback.format_tb(tb))
             out_queue.put((item, ret))
-    except Exception, err:
+    except Exception:
         if logger:
             logger.exception('Error in worker')
     finally:

slogging/log_processor.py

@@ -13,35 +13,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ConfigParser import ConfigParser
-import zlib
-import time
-import datetime
-import cStringIO
 import collections
-from paste.deploy import appconfig
-import multiprocessing
-import Queue
 import cPickle
+import cStringIO
+import datetime
 import hashlib
-from tzlocal import get_localzone
-import json
 import io
-
-from slogging.internal_proxy import InternalProxy
-from swift.common.utils import get_logger, readconf
-from swift.common.daemon import Daemon
-from slogging.log_common import LogProcessorCommon, multiprocess_collate, \
-                                   BadFileDownload
+import json
 from slogging import common
+from slogging import log_common
+from swift.common.daemon import Daemon
+from swift.common import utils
+import time
+from tzlocal import get_localzone
+
 
 now = datetime.datetime.now
 local_zone = get_localzone()
 
 
-class LogProcessor(LogProcessorCommon):
-    """Load plugins, process logs"""
+class LogProcessor(log_common.LogProcessorCommon):
+    """LogProcessor class.
 
+    Load plugins, process logs
+    """
     def __init__(self, conf, logger):
         basic_conf = conf['log-processor']
         super(LogProcessor, self).__init__(basic_conf, logger, 'log-processor')
@@ -63,8 +58,8 @@ class LogProcessor(LogProcessorCommon):
 
     def process_one_file(self, plugin_name, account, container, object_name):
         self.logger.info(_('Processing %(obj)s with plugin "%(plugin)s"') %
-                    {'obj': '/'.join((account, container, object_name)),
-                     'plugin': plugin_name})
+                         {'obj': '/'.join((account, container, object_name)),
+                          'plugin': plugin_name})
         # get an iter of the object data
         compressed = object_name.endswith('.gz')
         stream = self.get_object_data(account, container, object_name,
@@ -121,16 +116,16 @@ class LogProcessor(LogProcessorCommon):
 
 
 class LogProcessorDaemon(Daemon):
-    """
-    Gather raw log data and farm proccessing to generate a csv that is
+    """Log Processor Daemon class.
+
+    Gather raw log data and farm processing to generate a csv that is
     uploaded to swift.
     """
-
     def __init__(self, conf):
         c = conf.get('log-processor')
         super(LogProcessorDaemon, self).__init__(c)
         self.total_conf = conf
-        self.logger = get_logger(c, log_route='log-processor')
+        self.logger = utils.get_logger(c, log_route='log-processor')
         self.log_processor = LogProcessor(conf, self.logger)
         self.lookback_hours = int(c.get('lookback_hours', '120'))
         self.lookback_window = int(c.get('lookback_window',
@@ -147,7 +142,8 @@ class LogProcessorDaemon(Daemon):
                                                   'format_type', 'csv')
 
     def get_lookback_interval(self):
-        """
+        """Get lookback interval.
+
         :returns: lookback_start, lookback_end.
 
             Both or just lookback_end can be None. Otherwise, returns strings
@@ -169,19 +165,17 @@ class LogProcessorDaemon(Daemon):
                 lookback_end = None
             else:
                 delta_window = datetime.timedelta(hours=self.lookback_window)
-                lookback_end = now(self.time_zone) - \
-                               delta_hours + \
-                               delta_window
+                lookback_end = now(self.time_zone) - delta_hours + delta_window
                 lookback_end = lookback_end.strftime('%Y%m%d%H')
         return lookback_start, lookback_end
 
     def get_processed_files_list(self):
-        """
-        :returns: a set of files that have already been processed or returns
-        None on error.
+        """Downloads the set from the stats account.
 
-            Downloads the set from the stats account. Creates an empty set if
-            the an existing file cannot be found.
+        Creates an empty set if the an existing file cannot be found.
+
+        :returns: a set of files that have already been processed or returns
+            None on error.
         """
         try:
             # Note: this file (or data set) will grow without bound.
@@ -191,16 +185,16 @@ class LogProcessorDaemon(Daemon):
             # There is not a good way to determine when an old entry should be
             # pruned (lookback_hours could be set to anything and could change)
             stream = self.log_processor.get_object_data(
-                         self.log_processor_account,
-                         self.log_processor_container,
-                         self.processed_files_filename,
-                         compressed=True)
+                self.log_processor_account,
+                self.log_processor_container,
+                self.processed_files_filename,
+                compressed=True)
             buf = '\n'.join(x for x in stream)
             if buf:
                 files = cPickle.loads(buf)
             else:
                 return None
-        except BadFileDownload, err:
+        except log_common.BadFileDownload as err:
             if err.status_code == 404:
                 files = set()
             else:
@@ -208,11 +202,11 @@ class LogProcessorDaemon(Daemon):
         return files
 
     def get_aggregate_data(self, processed_files, input_data):
-        """
-        Aggregates stats data by account/hour, summing as needed.
+        """Aggregates stats data by account/hour, summing as needed.
 
         :param processed_files: set of processed files
-        :param input_data: is the output from multiprocess_collate/the plugins.
+        :param input_data: is the output from
+            log_common.multiprocess_collate/the plugins.
 
         :returns: A dict containing data aggregated from the input_data
         passed in.
@@ -250,8 +244,7 @@ class LogProcessorDaemon(Daemon):
         return aggr_data
 
     def get_final_info(self, aggr_data):
-        """
-        Aggregates data from aggr_data based on the keylist mapping.
+        """Aggregates data from aggr_data based on the keylist mapping.
 
         :param aggr_data: The results of the get_aggregate_data function.
         :returns: a dict of further aggregated data
@@ -287,21 +280,22 @@ class LogProcessorDaemon(Daemon):
         return final_info
 
     def store_processed_files_list(self, processed_files):
-        """
-        Stores the proccessed files list in the stats account.
+        """Stores the proccessed files list in the stats account.
 
         :param processed_files: set of processed files
         """
 
         s = cPickle.dumps(processed_files, cPickle.HIGHEST_PROTOCOL)
         f = cStringIO.StringIO(s)
-        self.log_processor.internal_proxy.upload_file(f,
+        self.log_processor.internal_proxy.upload_file(
+            f,
             self.log_processor_account,
             self.log_processor_container,
             self.processed_files_filename)
 
     def get_output(self, final_info):
-        """
+        """Return data according to given format_type.
+
         :returns: a list of rows to appear in the csv file or
                   a dictionary to appear in the json file.
 
@@ -345,14 +339,11 @@ class LogProcessorDaemon(Daemon):
         return output
 
     def restructure_stats_dictionary(self, target_dict):
-        """
-        Restructure stats dictionary for json format.
+        """Restructure stats dictionary for json format.
 
         :param target_dict: dictionary of restructuring target
-
         :returns: restructured stats dictionary
         """
-
         account_stats = {}
         access_stats = {}
         account_stats_key_list = \
@@ -369,15 +360,13 @@ class LogProcessorDaemon(Daemon):
         return hourly_stats
 
     def store_output(self, output):
-        """
-        Takes the dictionary or the list of rows and stores a json/csv file of
-        the values in the stats account.
+        """Takes the dictionary or the list of rows.
 
-        :param output: a dictonary or a list of row
+        And stores a json/csv file of the values in the stats account.
 
+        :param output: a dictionary or a list of row
             This json or csv file is final product of this script.
         """
-
         if self.format_type == 'json':
             out_buf = json.dumps(output, indent=2)
             h = hashlib.md5(out_buf).hexdigest()
@@ -390,32 +379,27 @@ class LogProcessorDaemon(Daemon):
             upload_name = datetime.datetime.now(self.time_zone).strftime(
                 '%Y/%m/%d/%H/') + '%s.csv.gz' % h
             f = cStringIO.StringIO(out_buf)
-        self.log_processor.internal_proxy.upload_file(f,
+        self.log_processor.internal_proxy.upload_file(
+            f,
             self.log_processor_account,
             self.log_processor_container,
             upload_name)
 
     @property
     def keylist_mapping(self):
-        """
-        :returns: the keylist mapping.
-
-            The keylist mapping determines how the stats fields are aggregated
-            in the final aggregation step.
-        """
-
-        if self._keylist_mapping == None:
+        """Determines how the stats fields are aggregated in the fila step."""
+        if self._keylist_mapping is None:
             self._keylist_mapping = \
                 self.log_processor.generate_keylist_mapping()
         return self._keylist_mapping
 
     def process_logs(self, logs_to_process, processed_files):
-        """
+        """Process logs and returns result as list.
+
         :param logs_to_process: list of logs to process
         :param processed_files: set of processed files
 
         :returns: returns a list of rows of processed data.
-
             The first row is the column headers. The rest of the rows contain
             hourly aggregate data for the account specified in the row.
 
@@ -427,9 +411,12 @@ class LogProcessorDaemon(Daemon):
 
         # map
         processor_args = (self.total_conf, self.logger)
-        results = multiprocess_collate(LogProcessor, processor_args,
-                                       'process_one_file', logs_to_process,
-                                       self.worker_count)
+        results = log_common.multiprocess_collate(
+            LogProcessor,
+            processor_args,
+            'process_one_file',
+            logs_to_process,
+            self.worker_count)
 
         # reduce
         aggr_data = self.get_aggregate_data(processed_files, results)
@@ -445,14 +432,11 @@ class LogProcessorDaemon(Daemon):
         return self.get_output(final_info)
 
     def run_once(self, *args, **kwargs):
-        """
-        Process log files that fall within the lookback interval.
+        """Process log files that fall within the lookback interval.
 
         Upload resulting csv or json file to stats account.
-
         Update processed files list and upload to stats account.
         """
-
         for k in 'lookback_hours lookback_window'.split():
             if k in kwargs and kwargs[k] is not None:
                 setattr(self, k, kwargs[k])
@@ -465,17 +449,18 @@ class LogProcessorDaemon(Daemon):
         self.logger.debug('lookback_end: %s' % lookback_end)
 
         processed_files = self.get_processed_files_list()
-        if processed_files == None:
+        if processed_files is None:
             self.logger.error(_('Log processing unable to load list of '
-                'already processed log files'))
+                                'already processed log files'))
             return
         self.logger.debug(_('found %d processed files') %
-            len(processed_files))
+                          len(processed_files))
 
-        logs_to_process = self.log_processor.get_data_list(lookback_start,
+        logs_to_process = self.log_processor.get_data_list(
+            lookback_start,
             lookback_end, processed_files)
         self.logger.info(_('loaded %d files to process') %
-            len(logs_to_process))
+                         len(logs_to_process))
 
         if logs_to_process:
             output = self.process_logs(logs_to_process, processed_files)
@@ -485,4 +470,4 @@ class LogProcessorDaemon(Daemon):
             self.store_processed_files_list(processed_files)
 
         self.logger.info(_("Log processing done (%0.2f minutes)") %
-            ((time.time() - start) / 60))
+                         ((time.time() - start) / 60))

slogging/log_uploader.py

@@ -14,25 +14,27 @@
 # limitations under the License.
 
 from __future__ import with_statement
-import os
-import hashlib
-import time
 import gzip
+import hashlib
+import os
+from paste.deploy import appconfig
+from paste.deploy import loadfilter
 import re
-import sys
-from paste.deploy import appconfig, loadfilter
-import zlib
-
 from slogging.internal_proxy import InternalProxy
 from swift.common.daemon import Daemon
 from swift.common import utils
+import sys
+import time
 
 
 class LogUploader(Daemon):
-    '''
-    Given a local directory, a swift account, and a container name, LogParser
-    will upload all files in the local directory to the given account/
-    container.  All but the newest files will be uploaded, and the files' md5
+    """LogUploader class.
+
+    Given a local directory, a swift account, and a container name,
+
+    LogParser will upload all files in the local directory to the given
+    account/container.
+    All but the newest files will be uploaded, and the files' md5
     sum will be computed. The hash is used to prevent duplicate data from
     being uploaded multiple times in different files (ex: log lines). Since
     the hash is computed, it is also used as the uploaded object's etag to
@@ -53,8 +55,7 @@ class LogUploader(Daemon):
         (?P<day>[0-3][0-9])
         (?P<hour>[0-2][0-9])
         .*$
-    '''
-
+    """
     def __init__(self, uploader_conf, plugin_name, regex=None, cutoff=None):
         super(LogUploader, self).__init__(uploader_conf)
         log_name = '%s-log-uploader' % plugin_name
@@ -63,8 +64,8 @@ class LogUploader(Daemon):
         self.log_dir = uploader_conf.get('log_dir', '/var/log/swift/')
         self.swift_account = uploader_conf['swift_account']
         self.container_name = uploader_conf['container_name']
-        proxy_server_conf_loc = uploader_conf.get('proxy_server_conf',
-                                            '/etc/swift/proxy-server.conf')
+        proxy_server_conf_loc = uploader_conf.get(
+            'proxy_server_conf', '/etc/swift/proxy-server.conf')
         proxy_server_conf = appconfig('config:%s' % proxy_server_conf_loc,
                                       name='proxy-server')
         memcache = loadfilter('config:%s' % proxy_server_conf_loc,
@@ -73,10 +74,12 @@ class LogUploader(Daemon):
                                             memcache=memcache)
         self.new_log_cutoff = int(cutoff or
                                   uploader_conf.get('new_log_cutoff', '7200'))
-        self.unlink_log = uploader_conf.get('unlink_log', 'true').lower() in \
-                utils.TRUE_VALUES
+        self.unlink_log = \
+            uploader_conf.get('unlink_log',
+                              'true').lower() in utils.TRUE_VALUES
         self.filename_pattern = regex or \
-            uploader_conf.get('source_filename_pattern',
+            uploader_conf.get(
+                'source_filename_pattern',
                 '''
                 ^%s-
                 (?P<year>[0-9]{4})
@@ -91,11 +94,10 @@ class LogUploader(Daemon):
         start = time.time()
         self.upload_all_logs()
         self.logger.info(_("Uploading logs complete (%0.2f minutes)") %
-            ((time.time() - start) / 60))
+                         ((time.time() - start) / 60))
 
     def get_relpath_to_files_under_log_dir(self):
-        """
-        Look under log_dir recursively and return all filenames as relpaths
+        """Look under log_dir recursively and return all filenames as relpaths
 
         :returns : list of strs, the relpath to all filenames under log_dir
         """
@@ -105,39 +107,36 @@ class LogUploader(Daemon):
         return [os.path.relpath(f, start=self.log_dir) for f in all_files]
 
     def filter_files(self, all_files):
-        """
-        Filter files based on regex pattern
+        """Filter files based on regex pattern.
 
         :param all_files: list of strs, relpath of the filenames under log_dir
         :param pattern: regex pattern to match against filenames
-
         :returns : dict mapping full path of file to match group dict
         """
         filename2match = {}
-        found_match = False
         for filename in all_files:
             match = re.match(self.filename_pattern, filename, re.VERBOSE)
             if match:
-                found_match = True
                 full_path = os.path.join(self.log_dir, filename)
                 filename2match[full_path] = match.groupdict()
             else:
                 self.logger.debug(_('%(filename)s does not match '
-                           '%(pattern)s') % {'filename': filename,
-                                             'pattern': self.filename_pattern})
+                                    '%(pattern)s') %
+                                  {'filename': filename,
+                                   'pattern': self.filename_pattern})
         return filename2match
 
     def upload_all_logs(self):
-        """
-        Match files under log_dir to source_filename_pattern and upload to
-        swift
+        """Match files under log_dir to source_filename_pattern.
+
+         And upload to swift
         """
         all_files = self.get_relpath_to_files_under_log_dir()
         filename2match = self.filter_files(all_files)
         if not filename2match:
             self.logger.error(_('No files in %(log_dir)s match %(pattern)s') %
-                     {'log_dir': self.log_dir,
-                      'pattern': self.filename_pattern})
+                              {'log_dir': self.log_dir,
+                               'pattern': self.filename_pattern})
             sys.exit(1)
         if not self.internal_proxy.create_container(self.swift_account,
                                                     self.container_name):
@@ -166,9 +165,7 @@ class LogUploader(Daemon):
                     _('ERROR: could not upload %s') % filename)
 
     def upload_one_log(self, filename, year, month, day, hour):
-        """
-        Upload one file to swift
-        """
+        """Upload one file to swift"""
         if os.path.getsize(filename) == 0:
             self.logger.debug(_("Log %s is 0 length, skipping") % filename)
             return
@@ -192,13 +189,13 @@ class LogUploader(Daemon):
         if self.content_type:
             metadata['Content-Type'] = self.content_type
         if self.internal_proxy.upload_file(filename,
-                                          self.swift_account,
-                                          self.container_name,
-                                          target_filename,
-                                          compress=(not already_compressed),
-                                          headers=metadata):
+                                           self.swift_account,
+                                           self.container_name,
+                                           target_filename,
+                                           compress=(not already_compressed),
+                                           headers=metadata):
             self.logger.debug(_("Uploaded log %(file)s to %(target)s") %
-                {'file': filename, 'target': target_filename})
+                              {'file': filename, 'target': target_filename})
             if self.unlink_log:
                 os.unlink(filename)
         else:

slogging/stats_processor.py

@@ -17,14 +17,16 @@ from swift.common.utils import get_logger
 
 
 class StatsLogProcessor(object):
-    """Transform account storage stat logs"""
+    """StatsLogProcessor class.
 
+    Transform account storage stat logs
+    """
     def __init__(self, conf):
         self.logger = get_logger(conf, log_route='stats-processor')
 
     def process(self, obj_stream, data_object_account, data_object_container,
                 data_object_name):
-        '''generate hourly groupings of data from one stats log file'''
+        """generate hourly groupings of data from one stats log file"""
         account_totals = {}
         year, month, day, hour, _junk = data_object_name.split('/')
         for line in obj_stream:
@@ -32,9 +34,9 @@ class StatsLogProcessor(object):
                 continue
             try:
                 (account,
-                container_count,
-                object_count,
-                bytes_used) = line.split(',')[:4]
+                 container_count,
+                 object_count,
+                 bytes_used) = line.split(',')[:4]
                 account = account.strip('"')
                 container_count = int(container_count.strip('"'))
                 object_count = int(object_count.strip('"'))
@@ -46,21 +48,17 @@ class StatsLogProcessor(object):
             aggr_key = (account, year, month, day, hour)
             d = account_totals.get(aggr_key, {})
             d['replica_count'] = d.setdefault('replica_count', 0) + 1
-            d['container_count'] = d.setdefault('container_count', 0) + \
-                                   container_count
-            d['object_count'] = d.setdefault('object_count', 0) + \
-                                object_count
-            d['bytes_used'] = d.setdefault('bytes_used', 0) + \
-                              bytes_used
+            d['container_count'] = \
+                d.setdefault('container_count', 0) + container_count
+            d['object_count'] = d.setdefault('object_count', 0) + object_count
+            d['bytes_used'] = d.setdefault('bytes_used', 0) + bytes_used
             account_totals[aggr_key] = d
         return account_totals
 
     def keylist_mapping(self):
-        '''
-        returns a dictionary of final keys mapped to source keys
-        '''
+        """Returns a dictionary of final keys mapped to source keys"""
         keylist_mapping = {
-        #   <db key> : <row key> or <set of row keys>
+            # <db key> : <row key> or <set of row keys>
             'bytes_used': 'bytes_used',
             'container_count': 'container_count',
             'object_count': 'object_count',

stdeb.cfg

@@ -1,5 +1,5 @@
 [DEFAULT]
 Provides: python-slogging
-Maintainer: John Dickinson <me@not.mn>
-Uploaders: John Dickinson <me@not.mn>
+Maintainer: Keiichi Hikita <keiichi.hikita@gmail.com>
+Uploaders: Keiichi Hikita <keiichi.hikita@gmail.com>
 Description: Stats and log processing services for swift.

test-requirements.txt

@@ -0,0 +1,13 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+
+# Hacking already pins down pep8, pyflakes and flake8
+hacking>=0.11.0,<0.12 # Apache-2.0
+coverage>=3.6 # Apache-2.0
+nose # LGPL
+nosexcover # BSD
+nosehtmloutput>=0.0.3 # Apache-2.0
+
+# Security checks
+bandit>=1.1.0 # Apache-2.0

test/sample.conf

@@ -0,0 +1,112 @@
+[func_test]
+# Sample config for Swift with tempauth
+auth_host = 127.0.0.1
+auth_port = 8080
+auth_ssl = no
+auth_prefix = /auth/
+# Sample config for Swift with Keystone v2 API.
+# For keystone v2 change auth_version to 2 and auth_prefix to /v2.0/.
+# And "allow_account_management" should not be set "true".
+#auth_version = 3
+#auth_host = localhost
+#auth_port = 5000
+#auth_ssl = no
+#auth_prefix = /v3/
+
+# Primary functional test account (needs admin access to the account)
+account = test
+username = tester
+password = testing
+
+# User on a second account (needs admin access to the account)
+account2 = test2
+username2 = tester2
+password2 = testing2
+
+# User on same account as first, but without admin access
+username3 = tester3
+password3 = testing3
+
+# Fourth user is required for keystone v3 specific tests.
+# Account must be in a non-default domain.
+#account4 = test4
+#username4 = tester4
+#password4 = testing4
+#domain4 = test-domain
+
+# Fifth user is required for service token-specific tests.
+# The account must be different from the primary test account.
+# The user must not have a group (tempauth) or role (keystoneauth) on
+# the primary test account. The user must have a group/role that is unique
+# and not given to the primary tester and is specified in the options
+# <prefix>_require_group (tempauth) or <prefix>_service_roles (keystoneauth).
+#account5 = test5
+#username5 = tester5
+#password5 = testing5
+
+# The service_prefix option is used for service token-specific tests.
+# If service_prefix or username5 above is not supplied, the tests are skipped.
+# To set the value and enable the service token tests, look at the
+# reseller_prefix option in /etc/swift/proxy-server.conf. There must be at
+# least two prefixes. If not, add a prefix as follows (where we add SERVICE):
+#     reseller_prefix = AUTH, SERVICE
+# The service_prefix must match the <prefix> used in <prefix>_require_group
+# (tempauth) or <prefix>_service_roles (keystoneauth); for example:
+#    SERVICE_require_group = service
+#    SERVICE_service_roles = service
+# Note: Do not enable service token tests if the first prefix in
+# reseller_prefix is the empty prefix AND the primary functional test
+# account contains an underscore.
+#service_prefix = SERVICE
+
+# Sixth user is required for access control tests.
+# Account must have a role for reseller_admin_role(keystoneauth).
+#account6 = test
+#username6 = tester6
+#password6 = testing6
+
+collate = C
+
+# Only necessary if a pre-existing server uses self-signed certificate
+insecure = no
+
+[unit_test]
+fake_syslog = False
+
+[probe_test]
+# check_server_timeout = 30
+# validate_rsync = false
+
+[swift-constraints]
+# The functional test runner will try to use the constraint values provided in
+# the swift-constraints section of test.conf.
+#
+# If a constraint value does not exist in that section, or because the
+# swift-constraints section does not exist, the constraints values found in
+# the /info API call (if successful) will be used.
+#
+# If a constraint value cannot be found in the /info results, either because
+# the /info API call failed, or a value is not present, the constraint value
+# used will fall back to those loaded by the constraints module at time of
+# import (which will attempt to load /etc/swift/swift.conf, see the
+# swift.common.constraints module for more information).
+#
+# Note that the cluster must have "sane" values for the test suite to pass
+# (for some definition of sane).
+#
+#max_file_size = 5368709122
+#max_meta_name_length = 128
+#max_meta_value_length = 256
+#max_meta_count = 90
+#max_meta_overall_size = 4096
+#max_header_size = 8192
+#extra_header_count = 0
+#max_object_name_length = 1024
+#container_listing_limit = 10000
+#account_listing_limit = 10000
+#max_account_name_length = 256
+#max_container_name_length = 256
+
+# Newer swift versions default to strict cors mode, but older ones were the
+# opposite.
+#strict_cors_mode = true

test/sample.proxy-server.conf

@@ -0,0 +1,944 @@
+[DEFAULT]
+# bind_ip = 0.0.0.0
+bind_port = 8080
+# bind_timeout = 30
+# backlog = 4096
+# swift_dir = /etc/swift
+# user = swift
+
+# Enables exposing configuration settings via HTTP GET /info.
+# expose_info = true
+
+# Key to use for admin calls that are HMAC signed.  Default is empty,
+# which will disable admin calls to /info.
+# admin_key = secret_admin_key
+#
+# Allows the ability to withhold sections from showing up in the public calls
+# to /info.  You can withhold subsections by separating the dict level with a
+# ".".  The following would cause the sections 'container_quotas' and 'tempurl'
+# to not be listed, and the key max_failed_deletes would be removed from
+# bulk_delete.  Default value is 'swift.valid_api_versions' which allows all
+# registered features to be listed via HTTP GET /info except
+# swift.valid_api_versions information
+# disallowed_sections = swift.valid_api_versions, container_quotas, tempurl
+
+# Use an integer to override the number of pre-forked processes that will
+# accept connections.  Should default to the number of effective cpu
+# cores in the system.  It's worth noting that individual workers will
+# use many eventlet co-routines to service multiple concurrent requests.
+# workers = auto
+#
+# Maximum concurrent requests per worker
+# max_clients = 1024
+#
+# Set the following two lines to enable SSL. This is for testing only.
+# cert_file = /etc/swift/proxy.crt
+# key_file = /etc/swift/proxy.key
+#
+# expiring_objects_container_divisor = 86400
+# expiring_objects_account_name = expiring_objects
+#
+# You can specify default log routing here if you want:
+# log_name = swift
+# log_facility = LOG_LOCAL0
+# log_level = INFO
+# log_headers = false
+# log_address = /dev/log
+# The following caps the length of log lines to the value given; no limit if
+# set to 0, the default.
+# log_max_line_length = 0
+#
+# This optional suffix (default is empty) that would be appended to the swift transaction
+# id allows one to easily figure out from which cluster that X-Trans-Id belongs to.
+# This is very useful when one is managing more than one swift cluster.
+# trans_id_suffix =
+#
+# comma separated list of functions to call to setup custom log handlers.
+# functions get passed: conf, name, log_to_console, log_route, fmt, logger,
+# adapted_logger
+# log_custom_handlers =
+#
+# If set, log_udp_host will override log_address
+# log_udp_host =
+# log_udp_port = 514
+#
+# You can enable StatsD logging here:
+# log_statsd_host =
+# log_statsd_port = 8125
+# log_statsd_default_sample_rate = 1.0
+# log_statsd_sample_rate_factor = 1.0
+# log_statsd_metric_prefix =
+#
+# Use a comma separated list of full URL (http://foo.bar:1234,https://foo.bar)
+# cors_allow_origin =
+# strict_cors_mode = True
+#
+# Comma separated list of headers to expose through Access-Control-Expose-Headers
+# cors_expose_headers =
+#
+# client_timeout = 60
+# eventlet_debug = false
+#
+# You can set scheduling priority of processes. Niceness values range from -20
+# (most favorable to the process) to 19 (least favorable to the process).
+# nice_priority =
+#
+# You can set I/O scheduling class and priority of processes. I/O niceness
+# class values are IOPRIO_CLASS_RT (realtime), IOPRIO_CLASS_BE (best-effort) and
+# IOPRIO_CLASS_IDLE (idle). I/O niceness priority is a number which goes from
+# 0 to 7. The higher the value, the lower the I/O priority of the process.
+# Work only with ionice_class.
+# ionice_class =
+# ionice_priority =
+
+[pipeline:main]
+# This sample pipeline uses tempauth and is used for SAIO dev work and
+# testing. See below for a pipeline using keystone.
+pipeline = catch_errors gatekeeper healthcheck proxy-logging cache listing_formats container_sync bulk tempurl ratelimit tempauth copy container-quotas account-quotas slo dlo versioned_writes symlink proxy-logging proxy-server
+
+# The following pipeline shows keystone integration. Comment out the one
+# above and uncomment this one. Additional steps for integrating keystone are
+# covered further below in the filter sections for authtoken and keystoneauth.
+#pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit authtoken keystoneauth copy container-quotas account-quotas slo dlo versioned_writes symlink proxy-logging proxy-server
+
+[app:proxy-server]
+use = egg:swift#proxy
+# You can override the default log routing for this app here:
+# set log_name = proxy-server
+# set log_facility = LOG_LOCAL0
+# set log_level = INFO
+# set log_address = /dev/log
+#
+# log_handoffs = true
+# recheck_account_existence = 60
+# recheck_container_existence = 60
+# object_chunk_size = 65536
+# client_chunk_size = 65536
+#
+# How long the proxy server will wait on responses from the a/c/o servers.
+# node_timeout = 10
+#
+# How long the proxy server will wait for an initial response and to read a
+# chunk of data from the object servers while serving GET / HEAD requests.
+# Timeouts from these requests can be recovered from so setting this to
+# something lower than node_timeout would provide quicker error recovery
+# while allowing for a longer timeout for non-recoverable requests (PUTs).
+# Defaults to node_timeout, should be overridden if node_timeout is set to a
+# high number to prevent client timeouts from firing before the proxy server
+# has a chance to retry.
+# recoverable_node_timeout = node_timeout
+#
+# conn_timeout = 0.5
+#
+# How long to wait for requests to finish after a quorum has been established.
+# post_quorum_timeout = 0.5
+#
+# How long without an error before a node's error count is reset. This will
+# also be how long before a node is reenabled after suppression is triggered.
+# error_suppression_interval = 60
+#
+# How many errors can accumulate before a node is temporarily ignored.
+# error_suppression_limit = 10
+#
+# If set to 'true' any authorized user may create and delete accounts; if
+# 'false' no one, even authorized, can.
+# allow_account_management = false
+#
+# If set to 'true' authorized accounts that do not yet exist within the Swift
+# cluster will be automatically created.
+# account_autocreate = false
+#
+# If set to a positive value, trying to create a container when the account
+# already has at least this maximum containers will result in a 403 Forbidden.
+# Note: This is a soft limit, meaning a user might exceed the cap for
+# recheck_account_existence before the 403s kick in.
+# max_containers_per_account = 0
+#
+# This is a comma separated list of account hashes that ignore the
+# max_containers_per_account cap.
+# max_containers_whitelist =
+#
+# Comma separated list of Host headers to which the proxy will deny requests.
+# deny_host_headers =
+#
+# Prefix used when automatically creating accounts.
+# auto_create_account_prefix = .
+#
+# Depth of the proxy put queue.
+# put_queue_depth = 10
+#
+# During GET and HEAD requests, storage nodes can be chosen at random
+# (shuffle), by using timing measurements (timing), or by using an explicit
+# region/zone match (affinity). Using timing measurements may allow for lower
+# overall latency, while using affinity allows for finer control. In both the
+# timing and affinity cases, equally-sorting nodes are still randomly chosen to
+# spread load.
+# The valid values for sorting_method are "affinity", "shuffle", or "timing".
+# This option may be overridden in a per-policy configuration section.
+# sorting_method = shuffle
+#
+# If the "timing" sorting_method is used, the timings will only be valid for
+# the number of seconds configured by timing_expiry.
+# timing_expiry = 300
+#
+# By default on a GET/HEAD swift will connect to a storage node one at a time
+# in a single thread. There is smarts in the order they are hit however. If you
+# turn on concurrent_gets below, then replica count threads will be used.
+# With addition of the concurrency_timeout option this will allow swift to send
+# out GET/HEAD requests to the storage nodes concurrently and answer with the
+# first to respond. With an EC policy the parameter only affects HEAD requests.
+# concurrent_gets = off
+#
+# This parameter controls how long to wait before firing off the next
+# concurrent_get thread. A value of 0 would be fully concurrent, any other
+# number will stagger the firing of the threads. This number should be
+# between 0 and node_timeout. The default is what ever you set for the
+# conn_timeout parameter.
+# concurrency_timeout = 0.5
+#
+# Set to the number of nodes to contact for a normal request. You can use
+# '* replicas' at the end to have it use the number given times the number of
+# replicas for the ring being used for the request.
+# request_node_count = 2 * replicas
+#
+# Specifies which backend servers to prefer on reads. Format is a comma
+# separated list of affinity descriptors of the form <selection>=<priority>.
+# The <selection> may be r<N> for selecting nodes in region N or r<N>z<M> for
+# selecting nodes in region N, zone M. The <priority> value should be a whole
+# number that represents the priority to be given to the selection; lower
+# numbers are higher priority.
+#
+# Example: first read from region 1 zone 1, then region 1 zone 2, then
+# anything in region 2, then everything else:
+# read_affinity = r1z1=100, r1z2=200, r2=300
+# Default is empty, meaning no preference.
+# This option may be overridden in a per-policy configuration section.
+# read_affinity =
+#
+# Specifies which backend servers to prefer on object writes. Format is a comma
+# separated list of affinity descriptors of the form r<N> for region N or
+# r<N>z<M> for region N, zone M. If this is set, then when handling an object
+# PUT request, some number (see setting write_affinity_node_count) of local
+# backend servers will be tried before any nonlocal ones.
+#
+# Example: try to write to regions 1 and 2 before writing to any other
+# nodes:
+# write_affinity = r1, r2
+# Default is empty, meaning no preference.
+# This option may be overridden in a per-policy configuration section.
+# write_affinity =
+#
+# The number of local (as governed by the write_affinity setting) nodes to
+# attempt to contact first on writes, before any non-local ones. The value
+# should be an integer number, or use '* replicas' at the end to have it use
+# the number given times the number of replicas for the ring being used for the
+# request.
+# This option may be overridden in a per-policy configuration section.
+# write_affinity_node_count = 2 * replicas
+#
+# The number of local (as governed by the write_affinity setting) handoff nodes
+# to attempt to contact on deletion, in addition to primary nodes.
+#
+# Example: in geographically distributed deployment of 2 regions, If
+# replicas=3, sometimes there may be 1 primary node and 2 local handoff nodes
+# in one region holding the object after uploading but before object replicated
+# to the appropriate locations in other regions. In this case, include these
+# handoff nodes to send request when deleting object could help make correct
+# decision for the response. The default value 'auto' means Swift will
+# calculate the number automatically, the default value is
+# (replicas - len(local_primary_nodes)). This option may be overridden in a
+# per-policy configuration section.
+# write_affinity_handoff_delete_count = auto
+#
+# These are the headers whose values will only be shown to swift_owners. The
+# exact definition of a swift_owner is up to the auth system in use, but
+# usually indicates administrative responsibilities.
+# swift_owner_headers = x-container-read, x-container-write, x-container-sync-key, x-container-sync-to, x-account-meta-temp-url-key, x-account-meta-temp-url-key-2, x-container-meta-temp-url-key, x-container-meta-temp-url-key-2, x-account-access-control
+#
+# You can set scheduling priority of processes. Niceness values range from -20
+# (most favorable to the process) to 19 (least favorable to the process).
+# nice_priority =
+#
+# You can set I/O scheduling class and priority of processes. I/O niceness
+# class values are IOPRIO_CLASS_RT (realtime), IOPRIO_CLASS_BE (best-effort) and
+# IOPRIO_CLASS_IDLE (idle). I/O niceness priority is a number which goes from
+# 0 to 7. The higher the value, the lower the I/O priority of the process.
+# Work only with ionice_class.
+# ionice_class =
+# ionice_priority =
+
+# Some proxy-server configuration options may be overridden on a per-policy
+# basis by including per-policy config section(s). The value of any option
+# specified a per-policy section will override any value given in the
+# proxy-server section for that policy only. Otherwise the value of these
+# options will be that specified in the proxy-server section.
+# The section name should refer to the policy index, not the policy name.
+# [proxy-server:policy:<policy index>]
+# sorting_method =
+# read_affinity =
+# write_affinity =
+# write_affinity_node_count =
+# write_affinity_handoff_delete_count =
+
+[filter:tempauth]
+use = egg:swift#tempauth
+# You can override the default log routing for this filter here:
+# set log_name = tempauth
+# set log_facility = LOG_LOCAL0
+# set log_level = INFO
+# set log_headers = false
+# set log_address = /dev/log
+#
+# The reseller prefix will verify a token begins with this prefix before even
+# attempting to validate it. Also, with authorization, only Swift storage
+# accounts with this prefix will be authorized by this middleware. Useful if
+# multiple auth systems are in use for one Swift cluster.
+# The reseller_prefix may contain a comma separated list of items. The first
+# item is used for the token as mentioned above. If second and subsequent
+# items exist, the middleware will handle authorization for an account with
+# that prefix. For example, for prefixes "AUTH, SERVICE", a path of
+# /v1/SERVICE_account is handled the same as /v1/AUTH_account. If an empty
+# (blank) reseller prefix is required, it must be first in the list. Two
+# single quote characters indicates an empty (blank) reseller prefix.
+# reseller_prefix = AUTH
+
+#
+# The require_group parameter names a group that must be presented by
+# either X-Auth-Token or X-Service-Token. Usually this parameter is
+# used only with multiple reseller prefixes (e.g., SERVICE_require_group=blah).
+# By default, no group is needed. Do not use .admin.
+# require_group =
+
+# The auth prefix will cause requests beginning with this prefix to be routed
+# to the auth subsystem, for granting tokens, etc.
+# auth_prefix = /auth/
+# token_life = 86400
+#
+# This allows middleware higher in the WSGI pipeline to override auth
+# processing, useful for middleware such as tempurl and formpost. If you know
+# you're not going to use such middleware and you want a bit of extra security,
+# you can set this to false.
+# allow_overrides = true
+#
+# This specifies what scheme to return with storage URLs:
+# http, https, or default (chooses based on what the server is running as)
+# This can be useful with an SSL load balancer in front of a non-SSL server.
+# storage_url_scheme = default
+#
+# Lastly, you need to list all the accounts/users you want here. The format is:
+#   user_<account>_<user> = <key> [group] [group] [...] [storage_url]
+# or if you want underscores in <account> or <user>, you can base64 encode them
+# (with no equal signs) and use this format:
+#   user64_<account_b64>_<user_b64> = <key> [group] [group] [...] [storage_url]
+# There are special groups of:
+#   .reseller_admin = can do anything to any account for this auth
+#   .admin = can do anything within the account
+# If neither of these groups are specified, the user can only access containers
+# that have been explicitly allowed for them by a .admin or .reseller_admin.
+# The trailing optional storage_url allows you to specify an alternate url to
+# hand back to the user upon authentication. If not specified, this defaults to
+# $HOST/v1/<reseller_prefix>_<account> where $HOST will do its best to resolve
+# to what the requester would need to use to reach this host.
+# Here are example entries, required for running the tests:
+user_admin_admin = admin .admin .reseller_admin
+user_test_tester = testing .admin
+user_test2_tester2 = testing2 .admin
+user_test_tester3 = testing3
+user_test5_tester5 = testing5 service
+
+# To enable Keystone authentication you need to have the auth token
+# middleware first to be configured. Here is an example below, please
+# refer to the keystone's documentation for details about the
+# different settings.
+#
+# You'll also need to have the keystoneauth middleware enabled and have it in
+# your main pipeline, as show in the sample pipeline at the top of this file.
+#
+# Following parameters are known to work with keystonemiddleware v2.3.0
+# (above v2.0.0), but checking the latest information in the wiki page[1]
+# is recommended.
+# 1. https://docs.openstack.org/keystonemiddleware/latest/middlewarearchitecture.html#configuration
+#
+# [filter:authtoken]
+# paste.filter_factory = keystonemiddleware.auth_token:filter_factory
+# auth_uri = http://keystonehost:5000
+# auth_url = http://keystonehost:35357
+# auth_plugin = password
+# The following credentials must match the Keystone credentials for the Swift
+# service and may need to be changed to match your Keystone configuration. The
+# example values shown here assume a user named 'swift' with admin role on a
+# project named 'service', both being in the Keystone domain with id 'default'.
+# Refer to the keystonemiddleware documentation link above [1] for other
+# examples.
+# project_domain_id = default
+# user_domain_id = default
+# project_name = service
+# username = swift
+# password = password
+#
+# delay_auth_decision defaults to False, but leaving it as false will
+# prevent other auth systems, staticweb, tempurl, formpost, and ACLs from
+# working. This value must be explicitly set to True.
+# delay_auth_decision = False
+#
+# cache = swift.cache
+# include_service_catalog = False
+#
+# [filter:keystoneauth]
+# use = egg:swift#keystoneauth
+# The reseller_prefix option lists account namespaces that this middleware is
+# responsible for. The prefix is placed before the Keystone project id.
+# For example, for project 12345678, and prefix AUTH, the account is
+# named AUTH_12345678 (i.e., path is /v1/AUTH_12345678/...).
+# Several prefixes are allowed by specifying a comma-separated list
+# as in: "reseller_prefix = AUTH, SERVICE". The empty string indicates a
+# single blank/empty prefix. If an empty prefix is required in a list of
+# prefixes, a value of '' (two single quote characters) indicates a
+# blank/empty prefix. Except for the blank/empty prefix, an underscore ('_')
+# character is appended to the value unless already present.
+# reseller_prefix = AUTH
+#
+# The user must have at least one role named by operator_roles on a
+# project in order to create, delete and modify containers and objects
+# and to set and read privileged headers such as ACLs.
+# If there are several reseller prefix items, you can prefix the
+# parameter so it applies only to those accounts (for example
+# the parameter SERVICE_operator_roles applies to the /v1/SERVICE_<project>
+# path). If you omit the prefix, the option applies to all reseller
+# prefix items. For the blank/empty prefix, prefix with '' (do not put
+# underscore after the two single quote characters).
+# operator_roles = admin, swiftoperator
+#
+# The reseller admin role has the ability to create and delete accounts
+# reseller_admin_role = ResellerAdmin
+#
+# This allows middleware higher in the WSGI pipeline to override auth
+# processing, useful for middleware such as tempurl and formpost. If you know
+# you're not going to use such middleware and you want a bit of extra security,
+# you can set this to false.
+# allow_overrides = true
+#
+# If the service_roles parameter is present, an X-Service-Token must be
+# present in the request that when validated, grants at least one role listed
+# in the parameter. The X-Service-Token may be scoped to any project.
+# If there are several reseller prefix items, you can prefix the
+# parameter so it applies only to those accounts (for example
+# the parameter SERVICE_service_roles applies to the /v1/SERVICE_<project>
+# path). If you omit the prefix, the option applies to all reseller
+# prefix items. For the blank/empty prefix, prefix with '' (do not put
+# underscore after the two single quote characters).
+# By default, no service_roles are required.
+# service_roles =
+#
+# For backwards compatibility, keystoneauth will match names in cross-tenant
+# access control lists (ACLs) when both the requesting user and the tenant
+# are in the default domain i.e the domain to which existing tenants are
+# migrated. The default_domain_id value configured here should be the same as
+# the value used during migration of tenants to keystone domains.
+# default_domain_id = default
+#
+# For a new installation, or an installation in which keystone projects may
+# move between domains, you should disable backwards compatible name matching
+# in ACLs by setting allow_names_in_acls to false:
+# allow_names_in_acls = true
+
+[filter:healthcheck]
+use = egg:swift#healthcheck
+# An optional filesystem path, which if present, will cause the healthcheck
+# URL to return "503 Service Unavailable" with a body of "DISABLED BY FILE".
+# This facility may be used to temporarily remove a Swift node from a load
+# balancer pool during maintenance or upgrade (remove the file to allow the
+# node back into the load balancer pool).
+# disable_path =
+
+[filter:cache]
+use = egg:swift#memcache
+# You can override the default log routing for this filter here:
+# set log_name = cache
+# set log_facility = LOG_LOCAL0
+# set log_level = INFO
+# set log_headers = false
+# set log_address = /dev/log
+#
+# If not set here, the value for memcache_servers will be read from
+# memcache.conf (see memcache.conf-sample) or lacking that file, it will
+# default to the value below. You can specify multiple servers separated with
+# commas, as in: 10.1.2.3:11211,10.1.2.4:11211 (IPv6 addresses must
+# follow rfc3986 section-3.2.2, i.e. [::1]:11211)
+# memcache_servers = 127.0.0.1:11211
+#
+# Sets how memcache values are serialized and deserialized:
+# 0 = older, insecure pickle serialization
+# 1 = json serialization but pickles can still be read (still insecure)
+# 2 = json serialization only (secure and the default)
+# If not set here, the value for memcache_serialization_support will be read
+# from /etc/swift/memcache.conf (see memcache.conf-sample).
+# To avoid an instant full cache flush, existing installations should
+# upgrade with 0, then set to 1 and reload, then after some time (24 hours)
+# set to 2 and reload.
+# In the future, the ability to use pickle serialization will be removed.
+# memcache_serialization_support = 2
+#
+# Sets the maximum number of connections to each memcached server per worker
+# memcache_max_connections = 2
+#
+# More options documented in memcache.conf-sample
+
+[filter:ratelimit]
+use = egg:swift#ratelimit
+# You can override the default log routing for this filter here:
+# set log_name = ratelimit
+# set log_facility = LOG_LOCAL0
+# set log_level = INFO
+# set log_headers = false
+# set log_address = /dev/log
+#
+# clock_accuracy should represent how accurate the proxy servers' system clocks
+# are with each other. 1000 means that all the proxies' clock are accurate to
+# each other within 1 millisecond.  No ratelimit should be higher than the
+# clock accuracy.
+# clock_accuracy = 1000
+#
+# max_sleep_time_seconds = 60
+#
+# log_sleep_time_seconds of 0 means disabled
+# log_sleep_time_seconds = 0
+#
+# allows for slow rates (e.g. running up to 5 sec's behind) to catch up.
+# rate_buffer_seconds = 5
+#
+# account_ratelimit of 0 means disabled
+# account_ratelimit = 0
+
+# DEPRECATED- these will continue to work but will be replaced
+# by the X-Account-Sysmeta-Global-Write-Ratelimit flag.
+# Please see ratelimiting docs for details.
+# these are comma separated lists of account names
+# account_whitelist = a,b
+# account_blacklist = c,d
+
+# with container_limit_x = r
+# for containers of size x limit write requests per second to r.  The container
+# rate will be linearly interpolated from the values given. With the values
+# below, a container of size 5 will get a rate of 75.
+# container_ratelimit_0 = 100
+# container_ratelimit_10 = 50
+# container_ratelimit_50 = 20
+
+# Similarly to the above container-level write limits, the following will limit
+# container GET (listing) requests.
+# container_listing_ratelimit_0 = 100
+# container_listing_ratelimit_10 = 50
+# container_listing_ratelimit_50 = 20
+
+[filter:domain_remap]
+use = egg:swift#domain_remap
+# You can override the default log routing for this filter here:
+# set log_name = domain_remap
+# set log_facility = LOG_LOCAL0
+# set log_level = INFO
+# set log_headers = false
+# set log_address = /dev/log
+#
+# Specify the storage_domain that match your cloud, multiple domains
+# can be specified separated by a comma
+# storage_domain = example.com
+
+# Specify a root path part that will be added to the start of paths if not
+# already present.
+# path_root = v1
+
+# Browsers can convert a host header to lowercase, so check that reseller
+# prefix on the account is the correct case. This is done by comparing the
+# items in the reseller_prefixes config option to the found prefix. If they
+# match except for case, the item from reseller_prefixes will be used
+# instead of the found reseller prefix. When none match, the default reseller
+# prefix is used. When no default reseller prefix is configured, any request
+# with an account prefix not in that list will be ignored by this middleware.
+# reseller_prefixes = AUTH
+# default_reseller_prefix =
+
+# Enable legacy remapping behavior for versioned path requests:
+#   c.a.example.com/v1/o -> /v1/AUTH_a/c/o
+# instead of
+#   c.a.example.com/v1/o -> /v1/AUTH_a/c/v1/o
+# ... by default all path parts after a remapped domain are considered part of
+# the object name with no special case for the path "v1"
+# mangle_client_paths = False
+
+[filter:catch_errors]
+use = egg:swift#catch_errors
+# You can override the default log routing for this filter here:
+# set log_name = catch_errors
+# set log_facility = LOG_LOCAL0
+# set log_level = INFO
+# set log_headers = false
+# set log_address = /dev/log
+
+[filter:cname_lookup]
+# Note: this middleware requires python-dnspython
+use = egg:swift#cname_lookup
+# You can override the default log routing for this filter here:
+# set log_name = cname_lookup
+# set log_facility = LOG_LOCAL0
+# set log_level = INFO
+# set log_headers = false
+# set log_address = /dev/log
+#
+# Specify the storage_domain that match your cloud, multiple domains
+# can be specified separated by a comma
+# storage_domain = example.com
+#
+# lookup_depth = 1
+#
+# Specify the nameservers to use to do the CNAME resolution. If unset, the
+# system configuration is used. Multiple nameservers can be specified
+# separated by a comma. Default port 53 can be overriden. IPv6 is accepted.
+# Example: 127.0.0.1, 127.0.0.2, 127.0.0.3:5353, [::1], [::1]:5353
+# nameservers =
+
+# Note: Put staticweb just after your auth filter(s) in the pipeline
+[filter:staticweb]
+use = egg:swift#staticweb
+# You can override the default log routing for this filter here:
+# set log_name = staticweb
+# set log_facility = LOG_LOCAL0
+# set log_level = INFO
+# set log_headers = false
+# set log_address = /dev/log
+#
+# At times when it's impossible for staticweb to guess the outside
+# endpoint correctly, the url_base may be used to supply the URL
+# scheme and/or the host name (and port number) in order to generate
+# redirects.
+# Example values:
+#    http://www.example.com    - redirect to www.example.com
+#    https:                    - changes the schema only
+#    https://                  - same, changes the schema only
+#    //www.example.com:8080    - redirect www.example.com on port 8080
+#                                (schema unchanged)
+# url_base =
+
+# Note: Put tempurl before dlo, slo and your auth filter(s) in the pipeline
+[filter:tempurl]
+use = egg:swift#tempurl
+# The methods allowed with Temp URLs.
+# methods = GET HEAD PUT POST DELETE
+#
+# The headers to remove from incoming requests. Simply a whitespace delimited
+# list of header names and names can optionally end with '*' to indicate a
+# prefix match. incoming_allow_headers is a list of exceptions to these
+# removals.
+# incoming_remove_headers = x-timestamp
+#
+# The headers allowed as exceptions to incoming_remove_headers. Simply a
+# whitespace delimited list of header names and names can optionally end with
+# '*' to indicate a prefix match.
+# incoming_allow_headers =
+#
+# The headers to remove from outgoing responses. Simply a whitespace delimited
+# list of header names and names can optionally end with '*' to indicate a
+# prefix match. outgoing_allow_headers is a list of exceptions to these
+# removals.
+# outgoing_remove_headers = x-object-meta-*
+#
+# The headers allowed as exceptions to outgoing_remove_headers. Simply a
+# whitespace delimited list of header names and names can optionally end with
+# '*' to indicate a prefix match.
+# outgoing_allow_headers = x-object-meta-public-*
+#
+# The digest algorithm(s) supported for generating signatures;
+# whitespace-delimited.
+# allowed_digests = sha1 sha256 sha512
+
+# Note: Put formpost just before your auth filter(s) in the pipeline
+[filter:formpost]
+use = egg:swift#formpost
+
+# Note: Just needs to be placed before the proxy-server in the pipeline.
+[filter:name_check]
+use = egg:swift#name_check
+# forbidden_chars = '"`<>
+# maximum_length = 255
+# forbidden_regexp = /\./|/\.\./|/\.$|/\.\.$
+
+[filter:list-endpoints]
+use = egg:swift#list_endpoints
+# list_endpoints_path = /endpoints/
+
+[filter:proxy-logging]
+use = egg:swift#proxy_logging
+# If not set, logging directives from [DEFAULT] without "access_" will be used
+# access_log_name = swift
+# access_log_facility = LOG_LOCAL0
+# access_log_level = INFO
+# access_log_address = /dev/log
+#
+# If set, access_log_udp_host will override access_log_address
+# access_log_udp_host =
+# access_log_udp_port = 514
+#
+# You can use log_statsd_* from [DEFAULT] or override them here:
+# access_log_statsd_host =
+# access_log_statsd_port = 8125
+# access_log_statsd_default_sample_rate = 1.0
+# access_log_statsd_sample_rate_factor = 1.0
+# access_log_statsd_metric_prefix =
+# access_log_headers = false
+#
+# If access_log_headers is True and access_log_headers_only is set only
+# these headers are logged. Multiple headers can be defined as comma separated
+# list like this: access_log_headers_only = Host, X-Object-Meta-Mtime
+# access_log_headers_only =
+#
+# By default, the X-Auth-Token is logged. To obscure the value,
+# set reveal_sensitive_prefix to the number of characters to log.
+# For example, if set to 12, only the first 12 characters of the
+# token appear in the log. An unauthorized access of the log file
+# won't allow unauthorized usage of the token. However, the first
+# 12 or so characters is unique enough that you can trace/debug
+# token usage. Set to 0 to suppress the token completely (replaced
+# by '...' in the log).
+# Note: reveal_sensitive_prefix will not affect the value
+# logged with access_log_headers=True.
+# reveal_sensitive_prefix = 16
+#
+# What HTTP methods are allowed for StatsD logging (comma-sep); request methods
+# not in this list will have "BAD_METHOD" for the <verb> portion of the metric.
+# log_statsd_valid_http_methods = GET,HEAD,POST,PUT,DELETE,COPY,OPTIONS
+#
+# Note: The double proxy-logging in the pipeline is not a mistake. The
+# left-most proxy-logging is there to log requests that were handled in
+# middleware and never made it through to the right-most middleware (and
+# proxy server). Double logging is prevented for normal requests. See
+# proxy-logging docs.
+
+# Note: Put before both ratelimit and auth in the pipeline.
+[filter:bulk]
+use = egg:swift#bulk
+# max_containers_per_extraction = 10000
+# max_failed_extractions = 1000
+# max_deletes_per_request = 10000
+# max_failed_deletes = 1000
+#
+# In order to keep a connection active during a potentially long bulk request,
+# Swift may return whitespace prepended to the actual response body. This
+# whitespace will be yielded no more than every yield_frequency seconds.
+# yield_frequency = 10
+#
+# Note: The following parameter is used during a bulk delete of objects and
+# their container. This would frequently fail because it is very likely
+# that all replicated objects have not been deleted by the time the middleware got a
+# successful response. It can be configured the number of retries. And the
+# number of seconds to wait between each retry will be 1.5**retry
+# delete_container_retry_count = 0
+#
+# To speed up the bulk delete process, multiple deletes may be executed in
+# parallel. Avoid setting this too high, as it gives clients a force multiplier
+# which may be used in DoS attacks. The suggested range is between 2 and 10.
+# delete_concurrency = 2
+
+# Note: Put after auth and staticweb in the pipeline.
+[filter:slo]
+use = egg:swift#slo
+# max_manifest_segments = 1000
+# max_manifest_size = 8388608
+#
+# Rate limiting applies only to segments smaller than this size (bytes).
+# rate_limit_under_size = 1048576
+#
+# Start rate-limiting SLO segment serving after the Nth small segment of a
+# segmented object.
+# rate_limit_after_segment = 10
+#
+# Once segment rate-limiting kicks in for an object, limit segments served
+# to N per second. 0 means no rate-limiting.
+# rate_limit_segments_per_sec = 1
+#
+# Time limit on GET requests (seconds)
+# max_get_time = 86400
+#
+# When creating an SLO, multiple segment validations may be executed in
+# parallel. Further, multiple deletes may be executed in parallel when deleting
+# with ?multipart-manifest=delete. Use this setting to limit how many
+# subrequests may be executed concurrently. Avoid setting it too high, as it
+# gives clients a force multiplier which may be used in DoS attacks. The
+# suggested range is between 2 and 10.
+# concurrency = 2
+#
+# This may be used to separately tune validation and delete concurrency values.
+# Default is to use the concurrency value from above; all of the same caveats
+# apply regarding recommended ranges.
+# delete_concurrency = 2
+#
+# In order to keep a connection active during a potentially long PUT request,
+# clients may request that Swift send whitespace ahead of the final response
+# body. This whitespace will be yielded at most every yield_frequency seconds.
+# yield_frequency = 10
+
+# Note: Put after auth and staticweb in the pipeline.
+# If you don't put it in the pipeline, it will be inserted for you.
+[filter:dlo]
+use = egg:swift#dlo
+# Start rate-limiting DLO segment serving after the Nth segment of a
+# segmented object.
+# rate_limit_after_segment = 10
+#
+# Once segment rate-limiting kicks in for an object, limit segments served
+# to N per second. 0 means no rate-limiting.
+# rate_limit_segments_per_sec = 1
+#
+# Time limit on GET requests (seconds)
+# max_get_time = 86400
+
+# Note: Put after auth in the pipeline.
+[filter:container-quotas]
+use = egg:swift#container_quotas
+
+# Note: Put after auth in the pipeline.
+[filter:account-quotas]
+use = egg:swift#account_quotas
+
+[filter:gatekeeper]
+use = egg:swift#gatekeeper
+# Set this to false if you want to allow clients to set arbitrary X-Timestamps
+# on uploaded objects. This may be used to preserve timestamps when migrating
+# from a previous storage system, but risks allowing users to upload
+# difficult-to-delete data.
+# shunt_inbound_x_timestamp = true
+#
+# You can override the default log routing for this filter here:
+# set log_name = gatekeeper
+# set log_facility = LOG_LOCAL0
+# set log_level = INFO
+# set log_headers = false
+# set log_address = /dev/log
+
+[filter:container_sync]
+use = egg:swift#container_sync
+# Set this to false if you want to disallow any full URL values to be set for
+# any new X-Container-Sync-To headers. This will keep any new full URLs from
+# coming in, but won't change any existing values already in the cluster.
+# Updating those will have to be done manually, as knowing what the true realm
+# endpoint should be cannot always be guessed.
+# allow_full_urls = true
+# Set this to specify this clusters //realm/cluster as "current" in /info
+# current = //REALM/CLUSTER
+
+# Note: Put it at the beginning of the pipeline to profile all middleware. But
+# it is safer to put this after catch_errors, gatekeeper and healthcheck.
+[filter:xprofile]
+use = egg:swift#xprofile
+# This option enable you to switch profilers which should inherit from python
+# standard profiler. Currently the supported value can be 'cProfile',
+# 'eventlet.green.profile' etc.
+# profile_module = eventlet.green.profile
+#
+# This prefix will be used to combine process ID and timestamp to name the
+# profile data file.  Make sure the executing user has permission to write
+# into this path (missing path segments will be created, if necessary).
+# If you enable profiling in more than one type of daemon, you must override
+# it with an unique value like: /var/log/swift/profile/proxy.profile
+# log_filename_prefix = /tmp/log/swift/profile/default.profile
+#
+# the profile data will be dumped to local disk based on above naming rule
+# in this interval.
+# dump_interval = 5.0
+#
+# Be careful, this option will enable profiler to dump data into the file with
+# time stamp which means there will be lots of files piled up in the directory.
+# dump_timestamp = false
+#
+# This is the path of the URL to access the mini web UI.
+# path = /__profile__
+#
+# Clear the data when the wsgi server shutdown.
+# flush_at_shutdown = false
+#
+# unwind the iterator of applications
+# unwind = false
+
+# Note: Put after slo, dlo in the pipeline.
+# If you don't put it in the pipeline, it will be inserted automatically.
+[filter:versioned_writes]
+use = egg:swift#versioned_writes
+# Enables using versioned writes middleware and exposing configuration
+# settings via HTTP GET /info.
+# WARNING: Setting this option bypasses the "allow_versions" option
+# in the container configuration file, which will be eventually
+# deprecated. See documentation for more details.
+# allow_versioned_writes = false
+
+# Note: Put after auth and before dlo and slo middlewares.
+# If you don't put it in the pipeline, it will be inserted for you.
+[filter:copy]
+use = egg:swift#copy
+
+# Note: To enable encryption, add the following 2 dependent pieces of crypto
+# middleware to the proxy-server pipeline. They should be to the right of all
+# other middleware apart from the final proxy-logging middleware, and in the
+# order shown in this example:
+# <other middleware> keymaster encryption proxy-logging proxy-server
+[filter:keymaster]
+use = egg:swift#keymaster
+
+# Sets the root secret from which encryption keys are derived. This must be set
+# before first use to a value that is a base64 encoding of at least 32 bytes.
+# The security of all encrypted data critically depends on this key, therefore
+# it should be set to a high-entropy value. For example, a suitable value may
+# be obtained by base-64 encoding a 32 byte (or longer) value generated by a
+# cryptographically secure random number generator. Changing the root secret is
+# likely to result in data loss.
+encryption_root_secret = changeme
+
+# Sets the path from which the keymaster config options should be read. This
+# allows multiple processes which need to be encryption-aware (for example,
+# proxy-server and container-sync) to share the same config file, ensuring
+# that the encryption keys used are the same. The format expected is similar
+# to other config files, with a single [keymaster] section and a single
+# encryption_root_secret option. If this option is set, the root secret
+# MUST NOT be set in proxy-server.conf.
+# keymaster_config_path =
+
+# To store the encryption root secret in a remote key management system (KMS)
+# such as Barbican, replace the keymaster middleware with the kms_keymaster
+# middleware in the proxy-server pipeline. They should be to the right of all
+# other middleware apart from the final proxy-logging middleware, and in the
+# order shown in this example:
+# <other middleware> kms_keymaster encryption proxy-logging proxy-server
+[filter:kms_keymaster]
+use = egg:swift#kms_keymaster
+
+# Sets the path from which the keymaster config options should be read. This
+# allows multiple processes which need to be encryption-aware (for example,
+# proxy-server and container-sync) to share the same config file, ensuring
+# that the encryption keys used are the same. The format expected is similar
+# to other config files, with a single [kms_keymaster] section. See the
+# keymaster.conf-sample file for details on the kms_keymaster configuration
+# options.
+# keymaster_config_path =
+
+[filter:encryption]
+use = egg:swift#encryption
+
+# By default all PUT or POST'ed object data and/or metadata will be encrypted.
+# Encryption of new data and/or metadata may be disabled by setting
+# disable_encryption to True. However, all encryption middleware should remain
+# in the pipeline in order for existing encrypted data to be read.
+# disable_encryption = False
+
+# listing_formats should be just right of the first proxy-logging middleware,
+# and left of most other middlewares. If it is not already present, it will
+# be automatically inserted for you.
+[filter:listing_formats]
+use = egg:swift#listing_formats
+
+# Note: Put after slo, dlo, versioned_writes, but before encryption in the
+# pipeline.
+[filter:symlink]
+use = egg:swift#symlink
+# Symlinks can point to other symlinks provided the number of symlinks in a
+# chain does not exceed the symloop_max value. If the number of chained
+# symlinks exceeds the limit symloop_max a 409 (HTTPConflict) error
+# response will be produced.
+# symloop_max = 2

test_slogging/speed/test_cidr_speed.py

@@ -1,10 +1,22 @@
-#!/usr/bin/env python
+# Copyright (c) 2010-2011 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
-import iptools
 import datetime
-
+import iptools
 import unittest
-from slogging import access_processor
 
 
 class TestAccessProcessorSpeed(unittest.TestCase):
@@ -14,14 +26,14 @@ class TestAccessProcessorSpeed(unittest.TestCase):
                '16/Sep/2012/20/00/02 GET /v1/a/c/o HTTP/1.0 ' \
                '200 - StaticWeb - - 17005 - txn - 0.0095 -'
         ips1 = iptools.IpRangeList(*[x.strip() for x in
-                                   '127.0.0.1,192.168/16,10/24'.split(',')
-                                   if x.strip()])
+                                     '127.0.0.1,192.168/16,10/24'.split(',')
+                                     if x.strip()])
         ips2 = iptools.IpRangeList(*[x.strip() for x in
-                                   '172.168/16,10/30'.split(',')
-                                   if x.strip()])
+                                     '172.168/16,10/30'.split(',')
+                                     if x.strip()])
         ips3 = iptools.IpRangeList(*[x.strip() for x in
-                                   '127.0.0.1,11/24'.split(',')
-                                   if x.strip()])
+                                     '127.0.0.1,11/24'.split(',')
+                                     if x.strip()])
 
         orig_start = datetime.datetime.utcnow()
         hit = 0

test_slogging/unit/__init__.py

@@ -1,25 +1,36 @@
-""" Swift tests """
+# Copyright (c) 2010-2011 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
-import sys
-import os
-import copy
-import logging
-from sys import exc_info
 from contextlib import contextmanager
-from tempfile import NamedTemporaryFile
+import copy
 from eventlet.green import socket
-from tempfile import mkdtemp
-from shutil import rmtree
-from ConfigParser import MissingSectionHeaderError
-from StringIO import StringIO
-from swift.common.utils import readconf, TRUE_VALUES
-from logging import Handler
+import logging
 import logging.handlers
+import os
+from shutil import rmtree
+from swift.common.utils import readconf
+from swift.common.utils import TRUE_VALUES
+import sys
+from sys import exc_info
+from tempfile import mkdtemp
+from tempfile import NamedTemporaryFile
 
 
 def get_config(section_name=None, defaults=None):
-    """
-    Attempt to get a test config dictionary.
+    """Attempt to get a test config dictionary.
 
     :param section_name: the section to read (all sections if not defined)
     :param defaults: an optional dictionary namespace of defaults
@@ -34,17 +45,17 @@ def get_config(section_name=None, defaults=None):
         config = readconf(config_file, section_name)
     except SystemExit:
         if not os.path.exists(config_file):
-            print >>sys.stderr, \
-                'Unable to read test config %s - file not found' \
-                % config_file
+            msg = 'Unable to read test config ' \
+                  '%s - file not found\n' % config_file
         elif not os.access(config_file, os.R_OK):
-            print >>sys.stderr, \
-                'Unable to read test config %s - permission denied' \
-                % config_file
+            msg = 'Unable to read test config %s - ' \
+                  'permission denied' % config_file
         else:
-            print >>sys.stderr, \
-                'Unable to read test config %s - section %s not found' \
-                % (config_file, section_name)
+            values = {'config_file': config_file,
+                      'section_name': section_name}
+            msg = 'Unable to read test config %(config_file)s - ' \
+                  'section %(section_name)s not found' % values
+        sys.stderr.write(msg)
     return config
 
 
@@ -169,8 +180,8 @@ class FakeLogger(object):
     def set_statsd_prefix(self, *a, **kw):
         pass
 
-    increment = decrement = timing = timing_since = update_stats = \
-            set_statsd_prefix
+    increment = decrement = timing = \
+        timing_since = update_stats = set_statsd_prefix
 
     def setFormatter(self, obj):
         self.formatter = obj
@@ -223,8 +234,8 @@ if get_config('unit_test').get('fake_syslog', 'False').lower() in TRUE_VALUES:
 
 
 class MockTrue(object):
-    """
-    Instances of MockTrue evaluate like True
+    """Instances of MockTrue evaluate like True
+
     Any attr accessed on an instance of MockTrue will return a MockTrue
     instance. Any method called on an instance of MockTrue will return
     a MockTrue instance.
@@ -248,9 +259,7 @@ class MockTrue(object):
     True
     >>> thing.method().attribute
     True
-
     """
-
     def __getattribute__(self, *args, **kwargs):
         return self
 

test_slogging/unit/test_access_log_delivery.py

@@ -13,16 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# TODO: Tests
 
-import unittest
 from contextlib import contextmanager
-
-from test_slogging.unit import temptree
-from swift.common import utils
-from slogging import access_log_delivery
-
 from nose.tools import nottest
+from slogging import access_log_delivery
+from swift.common import utils
+from test.unit import temptree
+import unittest
 
 
 class DumbLogger(object):
@@ -88,7 +85,7 @@ class TestAccessLogDelivery(unittest.TestCase):
                     'minute': '5', 'account': 'a', 'hour': '4',
                     'referrer': '9', 'request': '/v1/a/c/o?foo',
                     'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3'}
-        self.assertEquals(res, expected)
+        self.assertEqual(res, expected)
 
     def test_log_line_parser_hidden_ip(self):
         conf = {'hidden_ips': '1.2.3.4', 'swift_account': 'foo',
@@ -102,7 +99,7 @@ class TestAccessLogDelivery(unittest.TestCase):
         log_line = 'x' * 16 + ' '.join(log_line)
         res = p.log_line_parser(log_line)
         expected = '0.0.0.0'
-        self.assertEquals(res['client_ip'], expected)
+        self.assertEqual(res['client_ip'], expected)
         log_line = [str(x) for x in range(18)]
         log_line[1] = 'proxy-server'
         log_line[2] = '4.3.2.1'
@@ -111,7 +108,7 @@ class TestAccessLogDelivery(unittest.TestCase):
         log_line = 'x' * 16 + ' '.join(log_line)
         res = p.log_line_parser(log_line)
         expected = '4.3.2.1'
-        self.assertEquals(res['client_ip'], expected)
+        self.assertEqual(res['client_ip'], expected)
 
     def test_log_line_parser_field_count(self):
         p = access_log_delivery.AccessLogDelivery(self.conf, DumbLogger())
@@ -123,7 +120,7 @@ class TestAccessLogDelivery(unittest.TestCase):
         log_line = 'x' * 16 + ' '.join(log_line)
         res = p.log_line_parser(log_line)
         expected = {}
-        self.assertEquals(res, expected)
+        self.assertEqual(res, expected)
         # right amount of fields
         log_line = [str(x) for x in range(18)]
         log_line[1] = 'proxy-server'
@@ -139,7 +136,7 @@ class TestAccessLogDelivery(unittest.TestCase):
                     'minute': '5', 'account': 'a', 'hour': '4',
                     'referrer': '9', 'request': '/v1/a/c/o',
                     'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3'}
-        self.assertEquals(res, expected)
+        self.assertEqual(res, expected)
         # too many fields
         log_line = [str(x) for x in range(19)]
         log_line[1] = 'proxy-server'
@@ -148,7 +145,7 @@ class TestAccessLogDelivery(unittest.TestCase):
         log_line = 'x' * 16 + ' '.join(log_line)
         res = p.log_line_parser(log_line)
         # throws away invalid log lines
-        self.assertEquals(res, {})
+        self.assertEqual(res, {})
 
     def test_make_clf_from_parts(self):
         p = access_log_delivery.AccessLogDelivery(self.conf, DumbLogger())
@@ -160,7 +157,7 @@ class TestAccessLogDelivery(unittest.TestCase):
         parts = p.log_line_parser(log_line)
         clf = access_log_delivery.make_clf_from_parts(parts)
         expect = '2 - - [1/01/3:4:5:6 +0000] "5 /v1/a/c/o?foo 7" 8 13 "9" "10"'
-        self.assertEquals(clf, expect)
+        self.assertEqual(clf, expect)
 
     def test_convert_log_line(self):
         p = access_log_delivery.AccessLogDelivery(self.conf, DumbLogger())
@@ -174,7 +171,7 @@ class TestAccessLogDelivery(unittest.TestCase):
             '2 - - [1/01/3:4:5:6 +0000] "5 /v1/a/c/o?foo 7" 8 13 "9" "10"',
             'a',
             'c')
-        self.assertEquals(res, expected)
+        self.assertEqual(res, expected)
 
     # the following test fails, as it tries to load in /etc/swift/swift.conf
     @nottest
@@ -193,17 +190,17 @@ class TestAccessLogDelivery(unittest.TestCase):
         p.memcache = FakeMemcache()
         res = p.get_container_save_log_flag('a', 'c1')
         expected = False
-        self.assertEquals(res, expected)
+        self.assertEqual(res, expected)
         p.internal_proxy.get_container_metadata = my_get_metadata_true
         p.memcache = FakeMemcache()
         res = p.get_container_save_log_flag('a', 'c2')
         expected = True
-        self.assertEquals(res, expected)
+        self.assertEqual(res, expected)
         p.internal_proxy.get_container_metadata = my_get_metadata_true_upper
         p.memcache = FakeMemcache()
         res = p.get_container_save_log_flag('a', 'c3')
         expected = True
-        self.assertEquals(res, expected)
+        self.assertEqual(res, expected)
 
     def test_process_one_file(self):
         with temptree([]) as t:
@@ -212,7 +209,6 @@ class TestAccessLogDelivery(unittest.TestCase):
             p = access_log_delivery.AccessLogDelivery(conf, DumbLogger())
 
             def my_get_object_data(*a, **kw):
-                all_lines = []
                 log_line = [str(x) for x in range(18)]
                 log_line[1] = 'proxy-server'
                 log_line[4] = '1/Jan/3/4/5/6'
@@ -238,16 +234,16 @@ class TestAccessLogDelivery(unittest.TestCase):
             res = p.process_one_file('a', 'c', '2011/03/14/12/hash')
             expected = ['%s/a2/c2/2011/03/14/12' % t,
                         '%s/a/c/2011/03/14/12' % t]
-            self.assertEquals(res, set(expected))
+            self.assertEqual(res, set(expected))
             lines = [p.convert_log_line(x)[0] for x in my_get_object_data()]
             with open(expected[0], 'rb') as f:
                 raw = f.read()
                 res = '\n'.join(lines[2:]) + '\n'
-                self.assertEquals(res, raw)
+                self.assertEqual(res, raw)
             with open(expected[1], 'rb') as f:
                 raw = f.read()
                 res = '\n'.join(lines[:2]) + '\n'
-                self.assertEquals(res, raw)
+                self.assertEqual(res, raw)
 
 
 if __name__ == '__main__':

test_slogging/unit/test_access_processor.py

@@ -13,39 +13,38 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# TODO: Tests
 
-import unittest
 from slogging import access_processor
+import unittest
 
 
 class TestAccessProcessor(unittest.TestCase):
 
     def test_CIDR_works(self):
         if access_processor.CIDR_support:
-            p = access_processor.AccessLogProcessor({'lb_private_ips':
-                                                    '127.0.0.1,192.168/16,10/24'})
+            p = access_processor.AccessLogProcessor({
+                'lb_private_ips': '127.0.0.1,192.168/16,10/24'})
             self.assertTrue('192.168.2.3' in p.lb_private_ips)
             self.assertTrue('127.0.0.1' in p.lb_private_ips)
             self.assertFalse('192.167.2.3' in p.lb_private_ips)
         else:
             from nose import SkipTest
-            raise SkipTest("iptools for CIDR support not installed") 
+            raise SkipTest("iptools for CIDR support not installed")
 
     def test_CIDR_process_logs_with_missing_ip(self):
         if access_processor.CIDR_support:
-            p = access_processor.AccessLogProcessor({'lb_private_ips':
-                                                    '127.0.0.1,192.168/16,10/24',
-                                                    'server_name':'testsrv'})
+            p = access_processor.AccessLogProcessor({
+                'lb_private_ips': '127.0.0.1,192.168/16,10/24',
+                'server_name': 'testsrv'})
             line = 'Sep 16 20:00:02 srv testsrv 199.115.119.21 - ' \
-                     '16/Sep/2012/20/00/02 GET /v1/a/c/o HTTP/1.0 '  \
-                     '200 - StaticWeb - - 17005 - txn - 0.0095 -'
+                   '16/Sep/2012/20/00/02 GET /v1/a/c/o HTTP/1.0 '  \
+                   '200 - StaticWeb - - 17005 - txn - 0.0095 -'
             stream = [line]
             res = p.process(stream, 'dao', 'dac', 'don')
-            self.assertEquals(res.keys()[0][0], 'a')
+            self.assertEqual(res.keys()[0][0], 'a')
         else:
             from nose import SkipTest
-            raise SkipTest("iptools for CIDR support not installed") 
+            raise SkipTest("iptools for CIDR support not installed")
 
     def test_log_line_parser_query_args(self):
         p = access_processor.AccessLogProcessor({})
@@ -70,9 +69,10 @@ class TestAccessProcessor(unittest.TestCase):
         for param in access_processor.LISTING_PARAMS:
             expected[param] = 1
         expected['query'] = query
-        self.assertEquals(res, expected)
+        self.assertEqual(res, expected)
 
-    def test_log_line_parser_query_args_with_slash_delimiter_to_container(self):
+    def test_log_line_parser_query_args_with_slash_delimiter_to_container(
+            self):
         p = access_processor.AccessLogProcessor({})
         log_line = [str(x) for x in range(18)]
         log_line[1] = 'proxy-server'
@@ -82,11 +82,11 @@ class TestAccessProcessor(unittest.TestCase):
         log_line = 'x' * 16 + ' '.join(log_line)
         res = p.log_line_parser(log_line)
 
-        self.assertEquals(res['object_name'], None)
-        self.assertEquals(res['container_name'], 'c')
-        self.assertEquals(res['account'], 'a')
-        self.assertEquals(res['request'], '/v1/a/c')
-        self.assertEquals(res['query'], query)
+        self.assertEqual(res['object_name'], None)
+        self.assertEqual(res['container_name'], 'c')
+        self.assertEqual(res['account'], 'a')
+        self.assertEqual(res['request'], '/v1/a/c')
+        self.assertEqual(res['query'], query)
 
     def test_log_line_parser_query_args_with_slash_delimiter_to_account(self):
         p = access_processor.AccessLogProcessor({})
@@ -98,11 +98,11 @@ class TestAccessProcessor(unittest.TestCase):
         log_line = 'x' * 16 + ' '.join(log_line)
         res = p.log_line_parser(log_line)
 
-        self.assertEquals(res['object_name'], None)
-        self.assertEquals(res['container_name'], None)
-        self.assertEquals(res['account'], 'a')
-        self.assertEquals(res['request'], '/v1/a')
-        self.assertEquals(res['query'], query)
+        self.assertEqual(res['object_name'], None)
+        self.assertEqual(res['container_name'], None)
+        self.assertEqual(res['account'], 'a')
+        self.assertEqual(res['request'], '/v1/a')
+        self.assertEqual(res['query'], query)
 
     def test_log_line_parser_field_count(self):
         p = access_processor.AccessLogProcessor({})
@@ -114,7 +114,7 @@ class TestAccessProcessor(unittest.TestCase):
         log_line = 'x' * 16 + ' '.join(log_line)
         res = p.log_line_parser(log_line)
         expected = {}
-        self.assertEquals(res, expected)
+        self.assertEqual(res, expected)
         # right amount of fields
         log_line = [str(x) for x in range(18)]
         log_line[1] = 'proxy-server'
@@ -131,7 +131,7 @@ class TestAccessProcessor(unittest.TestCase):
                     'referrer': '9', 'request': '/v1/a/c/o',
                     'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3',
                     'log_source': None}
-        self.assertEquals(res, expected)
+        self.assertEqual(res, expected)
         # too many fields
         log_line = [str(x) for x in range(19)]
         log_line[1] = 'proxy-server'
@@ -148,7 +148,7 @@ class TestAccessProcessor(unittest.TestCase):
                     'referrer': '9', 'request': '/v1/a/c/o',
                     'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3',
                     'log_source': '18'}
-        self.assertEquals(res, expected)
+        self.assertEqual(res, expected)
 
 
 if __name__ == '__main__':

test_slogging/unit/test_compressing_file_reader.py

@@ -15,8 +15,8 @@
 
 """ Tests for swift.common.compressing_file_reader """
 
-import unittest
 import cStringIO
+import unittest
 
 from slogging.compressing_file_reader import CompressingFileReader
 
@@ -31,5 +31,5 @@ class TestCompressingFileReader(unittest.TestCase):
                    '\x00'
         x = CompressingFileReader(s)
         compressed = ''.join(iter(lambda: x.read(), ''))
-        self.assertEquals(compressed, expected)
-        self.assertEquals(x.read(), '')
+        self.assertEqual(compressed, expected)
+        self.assertEqual(x.read(), '')

test_slogging/unit/test_db_stats_collector.py

@@ -14,25 +14,25 @@
 # limitations under the License.
 
 
-import unittest
 import os
-import time
-import uuid
-import sqlite3
 from shutil import rmtree
 from slogging import db_stats_collector
-from tempfile import mkdtemp
-from test_slogging.unit import FakeLogger
+import sqlite3
 from swift.account.backend import AccountBroker
-from swift.container.backend import ContainerBroker
 from swift.common.utils import mkdirs
+from swift.container.backend import ContainerBroker
+from tempfile import mkdtemp
+from test.unit import FakeLogger
+import time
+import unittest
+import uuid
 
 
 class TestDbStats(unittest.TestCase):
 
     def setUp(self):
-        self._was_logger = db_stats_collector.get_logger
-        db_stats_collector.get_logger = FakeLogger
+        self._was_logger = db_stats_collector.utils.get_logger
+        db_stats_collector.utils.get_logger = FakeLogger
         self.testdir = os.path.join(mkdtemp(), 'tmp_test_db_stats')
         self.devices = os.path.join(self.testdir, 'node')
         rmtree(self.testdir, ignore_errors=1)
@@ -46,49 +46,25 @@ class TestDbStats(unittest.TestCase):
                          mount_check='false')
 
     def tearDown(self):
-        db_stats_collector.get_logger = self._was_logger
+        db_stats_collector.utils.get_logger = self._was_logger
         rmtree(self.testdir)
 
-    def test_account_stat_get_data(self):
-        stat = db_stats_collector.AccountStatsCollector(self.conf)
-        account_db = AccountBroker("%s/acc.db" % self.accounts,
-                                        account='test_acc')
-        account_db.initialize()
-        account_db.put_container('test_container', time.time(),
-                                      None, 10, 1000, 1)
-        info = stat.get_data("%s/acc.db" % self.accounts)
-        self.assertEquals('''"test_acc",1,10,1000\n''', info)
-
-    def test_container_stat_get_data(self):
-        stat = db_stats_collector.ContainerStatsCollector(self.conf)
-        container_db = ContainerBroker("%s/con.db" % self.containers,
-                                     account='test_acc', container='test_con')
-        container_db.initialize(storage_policy_index=0)
-        container_db.put_object('test_obj', time.time(), 10, 'text', 'faketag')
-        info = stat.get_data("%s/con.db" % self.containers)
-        self.assertEquals('''"test_acc","test_con",1,10\n''', info)
-
-    def test_container_stat_get_metadata(self):
-        container_db = ContainerBroker("%s/con.db" % self.containers,
-                                     account='test_acc', container='test_con')
-        container_db.initialize(storage_policy_index=0)
-        container_db.put_object('test_obj', time.time(), 10, 'text', 'faketag')
-        container_db.update_metadata({'X-Container-Meta-Test1': ('val', 1000)})
-        self.conf['metadata_keys'] = 'test1,test2'
-        stat = db_stats_collector.ContainerStatsCollector(self.conf)
-        info = stat.get_data("%s/con.db" % self.containers)
-        self.assertEquals('''"test_acc","test_con",1,10,1,\n''', info)
-
     def _gen_account_stat(self):
         stat = db_stats_collector.AccountStatsCollector(self.conf)
         output_data = set()
         for i in range(10):
-            account_db = AccountBroker("%s/stats-201001010%s-%s.db" %
-                                       (self.accounts, i, uuid.uuid4().hex),
-                                        account='test_acc_%s' % i)
+            account_db = AccountBroker(
+                "%s/stats-201001010%s-%s.db" % (self.accounts,
+                                                i,
+                                                uuid.uuid4().hex),
+                account='test_acc_%s' % i)
             account_db.initialize()
-            account_db.put_container('test_container', time.time(),
-                                      None, 10, 1000, 1)
+            account_db.put_container(name='test_container',
+                                     put_timestamp=time.time(),
+                                     delete_timestamp=0,
+                                     object_count=10,
+                                     bytes_used=1000,
+                                     storage_policy_index=1)
             # this will "commit" the data
             account_db.get_info()
             output_data.add('''"test_acc_%s",1,10,1000''' % i),
@@ -104,9 +80,10 @@ class TestDbStats(unittest.TestCase):
         output_data = set()
         for i in range(10):
             cont_db = ContainerBroker(
-                "%s/container-stats-201001010%s-%s.db" % (self.containers, i,
+                "%s/container-stats-201001010%s-%s.db" % (self.containers,
+                                                          i,
                                                           uuid.uuid4().hex),
-                 account='test_acc_%s' % i, container='test_con')
+                account='test_acc_%s' % i, container='test_con')
             cont_db.initialize(storage_policy_index=0)
             cont_db.put_object('test_obj', time.time(), 10, 'text', 'faketag')
             metadata_output = ''
@@ -125,6 +102,50 @@ class TestDbStats(unittest.TestCase):
         self.assertEqual(len(output_data), 10)
         return stat, output_data
 
+    def test_account_stat_get_data(self):
+        stat = db_stats_collector.AccountStatsCollector(self.conf)
+        account_db = AccountBroker("%s/acc.db" % self.accounts,
+                                   account='test_acc')
+        account_db.initialize()
+        account_db.put_container(name='test_container',
+                                 put_timestamp=time.time(),
+                                 delete_timestamp=0,
+                                 object_count=10,
+                                 bytes_used=1000,
+                                 storage_policy_index=1)
+        info = stat.get_data("%s/acc.db" % self.accounts)
+        self.assertEqual('''"test_acc",1,10,1000\n''', info)
+
+    def test_container_stat_get_data(self):
+        stat = db_stats_collector.ContainerStatsCollector(self.conf)
+        container_db = ContainerBroker("%s/con.db" % self.containers,
+                                       account='test_acc',
+                                       container='test_con')
+        container_db.initialize(storage_policy_index=0)
+        container_db.put_object(name='test_obj',
+                                timestamp=time.time(),
+                                size=10,
+                                content_type='text',
+                                etag='faketag')
+        info = stat.get_data("%s/con.db" % self.containers)
+        self.assertEqual('''"test_acc","test_con",1,10\n''', info)
+
+    def test_container_stat_get_metadata(self):
+        container_db = ContainerBroker("%s/con.db" % self.containers,
+                                       account='test_acc',
+                                       container='test_con')
+        container_db.initialize(storage_policy_index=0)
+        container_db.put_object(name='test_obj',
+                                timestamp=time.time(),
+                                size=10,
+                                content_type='text',
+                                etag='faketag')
+        container_db.update_metadata({'X-Container-Meta-Test1': ('val', 1000)})
+        self.conf['metadata_keys'] = 'test1,test2'
+        stat = db_stats_collector.ContainerStatsCollector(self.conf)
+        info = stat.get_data("%s/con.db" % self.containers)
+        self.assertEqual('''"test_acc","test_con",1,10,1,\n''', info)
+
     def test_account_stat_run_once_account(self):
         stat, output_data = self._gen_account_stat()
         stat.run_once()
@@ -142,7 +163,7 @@ class TestDbStats(unittest.TestCase):
         def raise_error(path):
             raise sqlite3.OperationalError('Test error')
         stat.get_data = raise_error
-        was_errors = len(stat.logger.log_dict['info'])
+        len(stat.logger.log_dict['info'])
         stat.run_once()
 
     def test_account_stat_run_once_container_metadata(self):
@@ -152,7 +173,7 @@ class TestDbStats(unittest.TestCase):
         stat_file = os.listdir(self.log_dir)[0]
         with open(os.path.join(self.log_dir, stat_file)) as stat_handle:
             headers = stat_handle.readline()
-            self.assert_(headers.startswith('Account Hash,Container Name,'))
+            self.assertTrue(headers.startswith('Account Hash,Container Name,'))
             for i in range(10):
                 data = stat_handle.readline()
                 output_data.discard(data.strip())
@@ -176,7 +197,7 @@ class TestDbStats(unittest.TestCase):
         stat_file = [f for f in os.listdir(self.log_dir) if f != stat_file][0]
         with open(os.path.join(self.log_dir, stat_file)) as stat_handle:
             headers = stat_handle.readline()
-            self.assert_(headers.startswith('Account Hash,Container Name,'))
+            self.assertTrue(headers.startswith('Account Hash,Container Name,'))
             for i in range(10):
                 data = stat_handle.readline()
                 con_output_data.discard(data.strip())
@@ -187,11 +208,11 @@ class TestDbStats(unittest.TestCase):
         stat, output_data = self._gen_account_stat()
         rmtree(self.accounts)
         stat.run_once()
-        self.assertEquals(len(stat.logger.log_dict['debug']), 1)
+        self.assertEqual(len(stat.logger.log_dict['debug']), 1)
 
     def test_not_implemented(self):
-        db_stat = db_stats_collector.DatabaseStatsCollector(self.conf,
-                                     'account', 'test_dir', 'stats-%Y%m%d%H_')
+        db_stat = db_stats_collector.DatabaseStatsCollector(
+            self.conf, 'account', 'test_dir', 'stats-%Y%m%d%H_')
         self.assertRaises(NotImplementedError, db_stat.get_data)
         self.assertRaises(NotImplementedError, db_stat.get_header)
 
@@ -199,7 +220,7 @@ class TestDbStats(unittest.TestCase):
         self.conf['mount_check'] = 'true'
         stat, output_data = self._gen_account_stat()
         stat.run_once()
-        self.assertEquals(len(stat.logger.log_dict['error']), 1)
+        self.assertEqual(len(stat.logger.log_dict['error']), 1)
 
 if __name__ == '__main__':
     unittest.main()

test_slogging/unit/test_internal_proxy.py

@@ -13,15 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# TODO: Tests
-
-import unittest
-import tempfile
 import json
-
-from swift.common import swob
-
 from slogging import internal_proxy
+from swift.common import swob
+import tempfile
+import unittest
 
 
 class DumbApplicationFactory(object):
@@ -60,7 +56,7 @@ class DumbApplication(object):
         else:
             body = self.body
         resp = swob.Response(request=req, body=body,
-                              conditional_response=True)
+                             conditional_response=True)
         try:
             resp.status_int = self.status_codes.pop(0)
         except IndexError:
@@ -77,79 +73,74 @@ class TestInternalProxy(unittest.TestCase):
     def test_swob_request_copy(self):
         req = swob.Request.blank('/')
         req2 = internal_proxy.swob_request_copy(req)
-        self.assertEquals(req.path, req2.path)
-        self.assertEquals(req.path_info, req2.path_info)
+        self.assertEqual(req.path, req2.path)
+        self.assertEqual(req.path_info, req2.path_info)
         self.assertFalse(req is req2)
-        self.assertEquals(req.headers, req2.headers)
+        self.assertEqual(req.headers, req2.headers)
         self.assertFalse(req.headers is req2.headers)
 
     def test_handle_request(self):
         status_codes = [200]
-        internal_proxy.Application = DumbApplicationFactory(
-                                            status_codes)
+        internal_proxy.Application = DumbApplicationFactory(status_codes)
         p = internal_proxy.InternalProxy()
         req = swob.Request.blank('/')
         orig_req = internal_proxy.swob_request_copy(req)
-        resp = p._handle_request(req)
-        self.assertEquals(req.path_info, orig_req.path_info)
+        p._handle_request(req)
+        self.assertEqual(req.path_info, orig_req.path_info)
 
     def test_handle_request_with_retries(self):
         status_codes = [500, 200]
-        internal_proxy.Application = DumbApplicationFactory(
-                                            status_codes)
+        internal_proxy.Application = DumbApplicationFactory(status_codes)
         p = internal_proxy.InternalProxy(retries=3)
         req = swob.Request.blank('/')
         orig_req = internal_proxy.swob_request_copy(req)
         resp = p._handle_request(req)
-        self.assertEquals(req.path_info, orig_req.path_info)
-        self.assertEquals(p.upload_app.call_count, 2)
-        self.assertEquals(resp.status_int, 200)
+        self.assertEqual(req.path_info, orig_req.path_info)
+        self.assertEqual(p.upload_app.call_count, 2)
+        self.assertEqual(resp.status_int, 200)
 
     def test_get_object(self):
         status_codes = [200]
-        internal_proxy.Application = DumbApplicationFactory(
-                                            status_codes)
+        internal_proxy.Application = DumbApplicationFactory(status_codes)
         p = internal_proxy.InternalProxy()
         code, body = p.get_object('a', 'c', 'o')
         body = ''.join(body)
-        self.assertEquals(code, 200)
-        self.assertEquals(body, '')
+        self.assertEqual(code, 200)
+        self.assertEqual(body, '')
 
     def test_create_container(self):
         status_codes = [200]
-        internal_proxy.Application = DumbApplicationFactory(
-                                            status_codes)
+        internal_proxy.Application = DumbApplicationFactory(status_codes)
         p = internal_proxy.InternalProxy()
         resp = p.create_container('a', 'c')
         self.assertTrue(resp)
 
     def test_handle_request_with_retries_all_error(self):
         status_codes = [500, 500, 500, 500, 500]
-        internal_proxy.Application = DumbApplicationFactory(
-                                            status_codes)
+        internal_proxy.Application = DumbApplicationFactory(status_codes)
         p = internal_proxy.InternalProxy(retries=3)
         req = swob.Request.blank('/')
         orig_req = internal_proxy.swob_request_copy(req)
         resp = p._handle_request(req)
-        self.assertEquals(req.path_info, orig_req.path_info)
-        self.assertEquals(p.upload_app.call_count, 3)
-        self.assertEquals(resp.status_int, 500)
+        self.assertEqual(req.path_info, orig_req.path_info)
+        self.assertEqual(p.upload_app.call_count, 3)
+        self.assertEqual(resp.status_int, 500)
 
     def test_get_container_list_empty(self):
         status_codes = [200]
-        internal_proxy.Application = DumbApplicationFactory(
-                                            status_codes, body='[]')
+        internal_proxy.Application = DumbApplicationFactory(status_codes,
+                                                            body='[]')
         p = internal_proxy.InternalProxy()
         resp = p.get_container_list('a', 'c')
-        self.assertEquals(resp, [])
+        self.assertEqual(resp, [])
 
     def test_get_container_list_no_body(self):
         status_codes = [204]
-        internal_proxy.Application = DumbApplicationFactory(
-                                            status_codes, body='')
+        internal_proxy.Application = DumbApplicationFactory(status_codes,
+                                                            body='')
         p = internal_proxy.InternalProxy()
         resp = p.get_container_list('a', 'c')
-        self.assertEquals(resp, [])
+        self.assertEqual(resp, [])
 
     def test_get_container_list_full_listing(self):
         status_codes = [200, 200]
@@ -158,26 +149,25 @@ class TestInternalProxy(unittest.TestCase):
         obj_b = dict(name='bar', hash='bar', bytes=3,
                      content_type='text/plain', last_modified='2011/01/01')
         body = [json.dumps([obj_a]), json.dumps([obj_b]), json.dumps([])]
-        internal_proxy.Application = DumbApplicationFactory(
-                                            status_codes, body=body)
+        internal_proxy.Application = DumbApplicationFactory(status_codes,
+                                                            body=body)
         p = internal_proxy.InternalProxy()
         resp = p.get_container_list('a', 'c')
         expected = ['foo', 'bar']
-        self.assertEquals([x['name'] for x in resp], expected)
+        self.assertEqual([x['name'] for x in resp], expected)
 
     def test_get_container_list_full(self):
         status_codes = [204]
-        internal_proxy.Application = DumbApplicationFactory(
-                                            status_codes, body='')
+        internal_proxy.Application = DumbApplicationFactory(status_codes,
+                                                            body='')
         p = internal_proxy.InternalProxy()
         resp = p.get_container_list('a', 'c', marker='a', end_marker='b',
                                     limit=100, prefix='/', delimiter='.')
-        self.assertEquals(resp, [])
+        self.assertEqual(resp, [])
 
     def test_upload_file(self):
         status_codes = [200, 200]  # container PUT + object PUT
-        internal_proxy.Application = DumbApplicationFactory(
-                                            status_codes)
+        internal_proxy.Application = DumbApplicationFactory(status_codes)
         p = internal_proxy.InternalProxy()
         with tempfile.NamedTemporaryFile() as file_obj:
             resp = p.upload_file(file_obj.name, 'a', 'c', 'o')
@@ -185,13 +175,13 @@ class TestInternalProxy(unittest.TestCase):
 
     def test_upload_file_with_retries(self):
         status_codes = [200, 500, 200]  # container PUT + error + object PUT
-        internal_proxy.Application = DumbApplicationFactory(
-                                            status_codes)
+        internal_proxy.Application = \
+            DumbApplicationFactory(status_codes)
         p = internal_proxy.InternalProxy(retries=3)
         with tempfile.NamedTemporaryFile() as file_obj:
             resp = p.upload_file(file_obj, 'a', 'c', 'o')
         self.assertTrue(resp)
-        self.assertEquals(p.upload_app.call_count, 3)
+        self.assertEqual(p.upload_app.call_count, 3)
 
 
 if __name__ == '__main__':

test_slogging/unit/test_log_common.py

@@ -1,8 +1,23 @@
-import unittest
+# Copyright (c) 2010-2011 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import Queue
 from slogging import log_common
-
 from slogging import log_processor
+from swift.common.exceptions import ChunkReadTimeout
+import unittest
 
 
 class DumbLogger(object):
@@ -58,14 +73,15 @@ class DumbInternalProxy(object):
 
 class TestLogProcessor(unittest.TestCase):
 
-    proxy_config = {'log-processor': {
-                    'swift_account': 'foo'}
-                   }
+    proxy_config = {
+        'log-processor': {
+            'swift_account': 'foo'
+        }}
     access_test_line = 'Jul  9 04:14:30 saio proxy-server 1.2.3.4 4.5.6.7 '\
-                    '09/Jul/2010/04/14/30 GET '\
-                    '/v1/acct/foo/bar?format=json&foo HTTP/1.0 200 - '\
-                    'curl tk4e350daf-9338-4cc6-aabb-090e49babfbd '\
-                    '6 95 - txfa431231-7f07-42fd-8fc7-7da9d8cc1f90 - 0.0262'
+                       '09/Jul/2010/04/14/30 GET '\
+                       '/v1/acct/foo/bar?format=json&foo HTTP/1.0 200 - '\
+                       'curl tk4e350daf-9338-4cc6-aabb-090e49babfbd '\
+                       '6 95 - txfa431231-7f07-42fd-8fc7-7da9d8cc1f90 - 0.0262'
 
     def test_collate_worker(self):
         try:
@@ -77,11 +93,12 @@ class TestLogProcessor(unittest.TestCase):
             log_processor.LogProcessor.get_object_data = get_object_data
             proxy_config = self.proxy_config.copy()
             proxy_config.update({
-                    'log-processor-access': {
-                        'source_filename_format': '%Y%m%d%H*',
-                        'class_path':
-                            'slogging.access_processor.AccessLogProcessor'
-                    }})
+                'log-processor-access': {
+                    'source_filename_format': '%Y%m%d%H*',
+                    'class_path':
+                        'slogging.access_processor.AccessLogProcessor',
+                    'time_zone': 'UTC',
+                }})
             processor_args = (proxy_config, DumbLogger())
             q_in = Queue.Queue()
             q_in.close = lambda: None
@@ -95,17 +112,18 @@ class TestLogProcessor(unittest.TestCase):
                                       'process_one_file', q_in, q_out,
                                       DumbLogger())
             item, ret = q_out.get()
-            self.assertEquals(item, work_request)
-            expected = {('acct', '2010', '07', '09', '04'):
-                        {('public', 'object', 'GET', '2xx'): 1,
-                        ('public', 'bytes_out'): 95,
-                        'marker_query': 0,
-                        'format_query': 1,
-                        'delimiter_query': 0,
-                        'path_query': 0,
-                        ('public', 'bytes_in'): 6,
-                        'prefix_query': 0}}
-            self.assertEquals(ret, expected)
+            self.assertEqual(item, work_request)
+            expected = {
+                ('acct', '2010', '07', '09', '04'):
+                    {('public', 'object', 'GET', '2xx'): 1,
+                     ('public', 'bytes_out'): 95,
+                     'marker_query': 0,
+                     'format_query': 1,
+                     'delimiter_query': 0,
+                     'path_query': 0,
+                     ('public', 'bytes_in'): 6,
+                     'prefix_query': 0}}
+            self.assertEqual(ret, expected)
         finally:
             log_processor.LogProcessor._internal_proxy = None
             log_processor.LogProcessor.get_object_data = orig_get_object_data
@@ -118,11 +136,12 @@ class TestLogProcessor(unittest.TestCase):
             log_processor.LogProcessor.get_object_data = get_object_data
             proxy_config = self.proxy_config.copy()
             proxy_config.update({
-                    'log-processor-access': {
-                        'source_filename_format': '%Y%m%d%H*',
-                        'class_path':
-                            'slogging.access_processor.AccessLogProcessor'
-                    }})
+                'log-processor-access': {
+                    'source_filename_format': '%Y%m%d%H*',
+                    'class_path':
+                        'slogging.access_processor.AccessLogProcessor',
+                    'time_zone': 'UTC',
+                }})
             processor_args = (proxy_config, DumbLogger())
             q_in = Queue.Queue()
             q_in.close = lambda: None
@@ -136,9 +155,9 @@ class TestLogProcessor(unittest.TestCase):
                                       'process_one_file', q_in, q_out,
                                       DumbLogger())
             item, ret = q_out.get()
-            self.assertEquals(item, work_request)
+            self.assertEqual(item, work_request)
             # these only work for Py2.7+
-            #self.assertIsInstance(ret, log_common.BadFileDownload)
+            # self.assertIsInstance(ret, log_common.BadFileDownload)
             self.assertTrue(isinstance(ret, Exception))
         finally:
             log_processor.LogProcessor.get_object_data = orig_get_object_data
@@ -153,32 +172,35 @@ class TestLogProcessor(unittest.TestCase):
             log_processor.LogProcessor.get_object_data = get_object_data
             proxy_config = self.proxy_config.copy()
             proxy_config.update({
-                    'log-processor-access': {
-                        'source_filename_format': '%Y%m%d%H*',
-                        'class_path':
-                            'slogging.access_processor.AccessLogProcessor'
-                    }})
+                'log-processor-access': {
+                    'source_filename_format': '%Y%m%d%H*',
+                    'class_path':
+                        'slogging.access_processor.AccessLogProcessor',
+                    'time_zone': 'UTC',
+                }})
             processor_args = (proxy_config, DumbLogger())
             item = ('access', 'a', 'c', 'o')
             logs_to_process = [item]
             processor_klass = log_processor.LogProcessor
-            results = log_processor.multiprocess_collate(processor_klass,
-                                                         processor_args,
-                                                         'process_one_file',
-                                                         logs_to_process,
-                                                         1,
-                                                         DumbLogger())
+            results = log_common.multiprocess_collate(
+                processor_klass,
+                processor_args,
+                'process_one_file',
+                logs_to_process,
+                1,
+                DumbLogger())
             results = list(results)
             expected = [(item, {('acct', '2010', '07', '09', '04'):
-                        {('public', 'object', 'GET', '2xx'): 1,
-                        ('public', 'bytes_out'): 95,
-                        'marker_query': 0,
-                        'format_query': 1,
-                        'delimiter_query': 0,
-                        'path_query': 0,
-                        ('public', 'bytes_in'): 6,
-                        'prefix_query': 0}})]
-            self.assertEquals(results, expected)
+                        {
+                            ('public', 'object', 'GET', '2xx'): 1,
+                            ('public', 'bytes_out'): 95,
+                            'marker_query': 0,
+                            'format_query': 1,
+                            'delimiter_query': 0,
+                            'path_query': 0,
+                            ('public', 'bytes_in'): 6,
+                            'prefix_query': 0}})]
+            self.assertEqual(results, expected)
         finally:
             log_processor.LogProcessor._internal_proxy = None
             log_processor.LogProcessor.get_object_data = orig_get_object_data
@@ -191,11 +213,12 @@ class TestLogProcessor(unittest.TestCase):
             log_processor.LogProcessor.get_object_data = get_object_data
             proxy_config = self.proxy_config.copy()
             proxy_config.update({
-                    'log-processor-access': {
-                        'source_filename_format': '%Y%m%d%H*',
-                        'class_path':
-                            'slogging.access_processor.AccessLogProcessor'
-                    }})
+                'log-processor-access': {
+                    'source_filename_format': '%Y%m%d%H*',
+                    'class_path':
+                        'slogging.access_processor.AccessLogProcessor',
+                    'time_zone': 'UTC',
+                }})
             processor_args = (proxy_config, DumbLogger())
             item = ('access', 'a', 'c', 'o')
             logs_to_process = [item]
@@ -208,7 +231,7 @@ class TestLogProcessor(unittest.TestCase):
                                                       DumbLogger())
             results = list(results)
             expected = []
-            self.assertEquals(results, expected)
+            self.assertEqual(results, expected)
         finally:
             log_processor.LogProcessor._internal_proxy = None
             log_processor.LogProcessor.get_object_data = orig_get_object_data

test_slogging/unit/test_log_processor.py

@@ -13,19 +13,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import unittest
-from test_slogging.unit import tmpfile
-import Queue
 import datetime
 import hashlib
-import pickle
-import time
 
+import pickle
 from slogging import internal_proxy
-from slogging import log_processor
 from slogging import log_common
+from slogging import log_processor
 from swift.common.exceptions import ChunkReadTimeout
 
+from test.unit import tmpfile
+import time
+import unittest
+
 
 class FakeUploadApp(object):
     def __init__(self, *args, **kwargs):
@@ -86,15 +86,16 @@ class DumbInternalProxy(object):
 class TestLogProcessor(unittest.TestCase):
 
     access_test_line = 'Jul  9 04:14:30 saio proxy-server 1.2.3.4 4.5.6.7 '\
-                    '09/Jul/2010/04/14/30 GET '\
-                    '/v1/acct/foo/bar?format=json&foo HTTP/1.0 200 - '\
-                    'curl tk4e350daf-9338-4cc6-aabb-090e49babfbd '\
-                    '6 95 - txfa431231-7f07-42fd-8fc7-7da9d8cc1f90 - 0.0262'
+                       '09/Jul/2010/04/14/30 GET '\
+                       '/v1/acct/foo/bar?format=json&foo HTTP/1.0 200 - '\
+                       'curl tk4e350daf-9338-4cc6-aabb-090e49babfbd '\
+                       '6 95 - txfa431231-7f07-42fd-8fc7-7da9d8cc1f90 - 0.0262'
     stats_test_line = 'account,1,2,3'
-    proxy_config = {'log-processor': {
-                    'swift_account': 'foo'
-                    }
-                   }
+    proxy_config = {
+        'log-processor': {
+            'swift_account': 'foo',
+            'time_zone': 'UTC',
+        }}
 
     def test_lazy_load_internal_proxy(self):
         # stub out internal_proxy's upload_app
@@ -103,29 +104,31 @@ class TestLogProcessor(unittest.TestCase):
 use = egg:swift#proxy
 """
         with tmpfile(dummy_proxy_config) as proxy_config_file:
-            conf = {'log-processor': {
+            conf = {
+                'log-processor': {
                     'proxy_server_conf': proxy_config_file,
-                    'swift_account': 'foo'
-                }
-            }
+                    'swift_account': 'foo',
+                    'time_zone': 'UTC',
+                }}
             p = log_processor.LogProcessor(conf, DumbLogger())
-            self.assert_(isinstance(p._internal_proxy,
-                                    None.__class__))
-            self.assert_(isinstance(p.internal_proxy,
-                                    log_processor.InternalProxy))
-            self.assertEquals(p.internal_proxy, p._internal_proxy)
+            self.assertTrue(isinstance(p._internal_proxy,
+                                       None.__class__))
+            self.assertTrue(isinstance(p.internal_proxy,
+                                       internal_proxy.InternalProxy))
+            self.assertEqual(p.internal_proxy, p._internal_proxy)
         # test with empty config variable
-        conf = {'log-processor': {
+        conf = {
+            'log-processor': {
                 'proxy_server_conf': '',
-                'swift_account': 'foo'
-            }
-        }
+                'swift_account': 'foo',
+                'time_zone': 'UTC',
+            }}
         q = log_processor.LogProcessor(conf, DumbLogger())
-        self.assert_(isinstance(q._internal_proxy,
-                                None.__class__))
-        self.assert_(isinstance(q.internal_proxy,
-                                log_processor.InternalProxy))
-        self.assertEquals(q.internal_proxy, q._internal_proxy)
+        self.assertTrue(isinstance(q._internal_proxy,
+                                   None.__class__))
+        self.assertTrue(isinstance(q.internal_proxy,
+                                   internal_proxy.InternalProxy))
+        self.assertEqual(q.internal_proxy, q._internal_proxy)
 
         # reset FakeUploadApp
         reload(internal_proxy)
@@ -133,75 +136,76 @@ use = egg:swift#proxy
     def test_access_log_line_parser(self):
         access_proxy_config = self.proxy_config.copy()
         access_proxy_config.update({
-                        'log-processor-access': {
-                            'source_filename_format': '%Y%m%d%H*',
-                            'class_path':
-                                'slogging.access_processor.AccessLogProcessor'
-                        }})
+            'log-processor-access': {
+                'source_filename_format': '%Y%m%d%H*',
+                'class_path': 'slogging.access_processor.AccessLogProcessor',
+                'time_zone': 'UTC',
+            }})
         p = log_processor.LogProcessor(access_proxy_config, DumbLogger())
         result = p.plugins['access']['instance'].log_line_parser(
-                                                    self.access_test_line)
-        self.assertEquals(result, {'code': 200,
-           'processing_time': '0.0262',
-           'auth_token': 'tk4e350daf-9338-4cc6-aabb-090e49babfbd',
-           'month': '07',
-           'second': '30',
-           'year': '2010',
-           'query': 'format=json&foo',
-           'tz': '+0000',
-           'http_version': 'HTTP/1.0',
-           'object_name': 'bar',
-           'etag': '-',
-           'method': 'GET',
-           'trans_id': 'txfa431231-7f07-42fd-8fc7-7da9d8cc1f90',
-           'client_ip': '1.2.3.4',
-           'format': 1,
-           'bytes_out': 95,
-           'container_name': 'foo',
-           'day': '09',
-           'minute': '14',
-           'account': 'acct',
-           'hour': '04',
-           'referrer': '-',
-           'request': '/v1/acct/foo/bar',
-           'user_agent': 'curl',
-           'bytes_in': 6,
-           'lb_ip': '4.5.6.7',
-           'log_source': None})
+            self.access_test_line)
+        self.assertEqual(result, {
+            'code': 200,
+            'processing_time': '0.0262',
+            'auth_token': 'tk4e350daf-9338-4cc6-aabb-090e49babfbd',
+            'month': '07',
+            'second': '30',
+            'year': '2010',
+            'query': 'format=json&foo',
+            'tz': '+0000',
+            'http_version': 'HTTP/1.0',
+            'object_name': 'bar',
+            'etag': '-',
+            'method': 'GET',
+            'trans_id': 'txfa431231-7f07-42fd-8fc7-7da9d8cc1f90',
+            'client_ip': '1.2.3.4',
+            'format': 1,
+            'bytes_out': 95,
+            'container_name': 'foo',
+            'day': '09',
+            'minute': '14',
+            'account': 'acct',
+            'hour': '04',
+            'referrer': '-',
+            'request': '/v1/acct/foo/bar',
+            'user_agent': 'curl',
+            'bytes_in': 6,
+            'lb_ip': '4.5.6.7',
+            'log_source': None})
 
     def test_process_one_access_file(self):
         access_proxy_config = self.proxy_config.copy()
         access_proxy_config.update({
-                        'log-processor-access': {
-                            'source_filename_format': '%Y%m%d%H*',
-                            'class_path':
-                                'slogging.access_processor.AccessLogProcessor'
-                        }})
+            'log-processor-access': {
+                'source_filename_format': '%Y%m%d%H*',
+                'class_path': 'slogging.access_processor.AccessLogProcessor',
+                'time_zone': 'UTC',
+            }})
         p = log_processor.LogProcessor(access_proxy_config, DumbLogger())
 
         def get_object_data(*a, **kw):
             return [self.access_test_line]
         p.get_object_data = get_object_data
         result = p.process_one_file('access', 'a', 'c', 'o')
-        expected = {('acct', '2010', '07', '09', '04'):
-                    {('public', 'object', 'GET', '2xx'): 1,
-                    ('public', 'bytes_out'): 95,
-                    'marker_query': 0,
-                    'format_query': 1,
-                    'delimiter_query': 0,
-                    'path_query': 0,
-                    ('public', 'bytes_in'): 6,
-                    'prefix_query': 0}}
-        self.assertEquals(result, expected)
+        expected = {('acct', '2010', '07', '09', '04'): {
+            ('public', 'object', 'GET', '2xx'): 1,
+            ('public', 'bytes_out'): 95,
+            'marker_query': 0,
+            'format_query': 1,
+            'delimiter_query': 0,
+            'path_query': 0,
+            ('public', 'bytes_in'): 6,
+            'prefix_query': 0}}
+        self.assertEqual(result, expected)
 
     def test_process_one_access_file_error(self):
         access_proxy_config = self.proxy_config.copy()
         access_proxy_config.update({
-                        'log-processor-access': {
-                            'source_filename_format': '%Y%m%d%H*',
-                            'class_path':
-                                'slogging.access_processor.AccessLogProcessor'
-                        }})
+            'log-processor-access': {
+                'source_filename_format': '%Y%m%d%H*',
+                'class_path': 'slogging.access_processor.AccessLogProcessor',
+                'time_zone': 'UTC',
+            }})
         p = log_processor.LogProcessor(access_proxy_config, DumbLogger())
         p._internal_proxy = DumbInternalProxy(code=500)
         self.assertRaises(log_common.BadFileDownload, p.process_one_file,
@@ -212,34 +216,34 @@ use = egg:swift#proxy
         p._internal_proxy = DumbInternalProxy()
         result = p.get_container_listing('a', 'foo')
         expected = ['2010/03/14/13/obj1']
-        self.assertEquals(result, expected)
+        self.assertEqual(result, expected)
         result = p.get_container_listing('a', 'foo', listing_filter=expected)
         expected = []
-        self.assertEquals(result, expected)
+        self.assertEqual(result, expected)
         result = p.get_container_listing('a', 'foo', start_date='2010031412',
-                                            end_date='2010031414')
+                                         end_date='2010031414')
         expected = ['2010/03/14/13/obj1']
-        self.assertEquals(result, expected)
+        self.assertEqual(result, expected)
         result = p.get_container_listing('a', 'foo', start_date='2010031414')
         expected = []
-        self.assertEquals(result, expected)
+        self.assertEqual(result, expected)
         result = p.get_container_listing('a', 'foo', start_date='2010031410',
-                                            end_date='2010031412')
+                                         end_date='2010031412')
         expected = []
-        self.assertEquals(result, expected)
+        self.assertEqual(result, expected)
         result = p.get_container_listing('a', 'foo', start_date='2010031412',
-                                            end_date='2010031413')
+                                         end_date='2010031413')
         expected = ['2010/03/14/13/obj1']
-        self.assertEquals(result, expected)
+        self.assertEqual(result, expected)
 
     def test_get_object_data(self):
         p = log_processor.LogProcessor(self.proxy_config, DumbLogger())
         p._internal_proxy = DumbInternalProxy()
         result = list(p.get_object_data('a', 'c', 'o', False))
         expected = ['obj', 'data']
-        self.assertEquals(result, expected)
+        self.assertEqual(result, expected)
         result = list(p.get_object_data('a', 'c', 'o.gz', True))
-        self.assertEquals(result, expected)
+        self.assertEqual(result, expected)
 
     def test_get_object_data_errors(self):
         p = log_processor.LogProcessor(self.proxy_config, DumbLogger())
@@ -256,10 +260,10 @@ use = egg:swift#proxy
     def test_get_stat_totals(self):
         stats_proxy_config = self.proxy_config.copy()
         stats_proxy_config.update({
-                        'log-processor-stats': {
-                            'class_path':
-                                'slogging.stats_processor.StatsLogProcessor'
-                        }})
+            'log-processor-stats': {
+                'class_path': 'slogging.stats_processor.StatsLogProcessor',
+                'time_zone': 'UTC',
+            }})
         p = log_processor.LogProcessor(stats_proxy_config, DumbLogger())
         p._internal_proxy = DumbInternalProxy()
 
@@ -267,23 +271,24 @@ use = egg:swift#proxy
             return [self.stats_test_line]
         p.get_object_data = get_object_data
         result = p.process_one_file('stats', 'a', 'c', 'y/m/d/h/o')
-        expected = {('account', 'y', 'm', 'd', 'h'):
-                    {'replica_count': 1,
-                    'object_count': 2,
-                    'container_count': 1,
-                    'bytes_used': 3}}
-        self.assertEquals(result, expected)
+        expected = {
+            ('account', 'y', 'm', 'd', 'h'):
+                {'replica_count': 1,
+                 'object_count': 2,
+                 'container_count': 1,
+                 'bytes_used': 3}}
+        self.assertEqual(result, expected)
 
     def test_generate_keylist_mapping(self):
         p = log_processor.LogProcessor(self.proxy_config, DumbLogger())
         result = p.generate_keylist_mapping()
         expected = {}
-        self.assertEquals(result, expected)
+        self.assertEqual(result, expected)
 
     def test_generate_keylist_mapping_with_dummy_plugins(self):
         class Plugin1(object):
             def keylist_mapping(self):
-                return {'a': 'b', 'c': 'd', 'e': ['f', 'g']}
+                return {'a': 'b', 'c': 'd', 'e': set(['f', 'g'])}
 
         class Plugin2(object):
             def keylist_mapping(self):
@@ -294,35 +299,36 @@ use = egg:swift#proxy
         result = p.generate_keylist_mapping()
         expected = {'a': set(['b', '1']), 'c': 'd', 'e': set(['2', 'f', 'g']),
                     'h': '3'}
-        self.assertEquals(result, expected)
+        self.assertEqual(result, expected)
 
     def test_access_keylist_mapping_format(self):
         proxy_config = self.proxy_config.copy()
         proxy_config.update({
-                        'log-processor-access': {
-                            'source_filename_format': '%Y%m%d%H*',
-                            'class_path':
-                                'slogging.access_processor.AccessLogProcessor'
-                        }})
+            'log-processor-access': {
+                'source_filename_format': '%Y%m%d%H*',
+                'class_path':
+                    'slogging.access_processor.AccessLogProcessor',
+                'time_zone': 'UTC',
+            }})
         p = log_processor.LogProcessor(proxy_config, DumbLogger())
         mapping = p.generate_keylist_mapping()
         for k, v in mapping.items():
             # these only work for Py2.7+
-            #self.assertIsInstance(k, str)
+            # self.assertIsInstance(k, str)
             self.assertTrue(isinstance(k, str), type(k))
 
     def test_stats_keylist_mapping_format(self):
         proxy_config = self.proxy_config.copy()
         proxy_config.update({
-                        'log-processor-stats': {
-                            'class_path':
-                                'slogging.stats_processor.StatsLogProcessor'
-                        }})
+            'log-processor-stats': {
+                'class_path': 'slogging.stats_processor.StatsLogProcessor',
+                'time_zone': 'UTC',
+            }})
         p = log_processor.LogProcessor(proxy_config, DumbLogger())
         mapping = p.generate_keylist_mapping()
         for k, v in mapping.items():
             # these only work for Py2.7+
-            #self.assertIsInstance(k, str)
+            # self.assertIsInstance(k, str)
             self.assertTrue(isinstance(k, str), type(k))
 
 
@@ -346,17 +352,16 @@ class TestLogProcessorDaemon(unittest.TestCase):
                 [d(2009, 5, 6, 7, 8), 1200, 100, '2009031707', '2009032111'],
                 [d(2008, 9, 10, 11, 12), 3000, 1000, '2008050811',
                     '2008061903'],
-                ]:
-
+            ]:
                 log_processor.now = lambda tz: x[0]
 
                 d = MockLogProcessorDaemon(x[1], x[2])
-                self.assertEquals((x[3], x[4]), d.get_lookback_interval())
+                self.assertEqual((x[3], x[4]), d.get_lookback_interval())
         finally:
             log_processor.now = datetime.datetime.now
 
     def test_get_processed_files_list(self):
-        class MockLogProcessor():
+        class MockLogProcessor(object):
             def __init__(self, stream):
                 self.stream = stream
 
@@ -372,16 +377,16 @@ class TestLogProcessorDaemon(unittest.TestCase):
 
         file_list = set(['a', 'b', 'c'])
 
-        for s, l in [['', None],
-                [pickle.dumps(set()).split('\n'), set()],
-                [pickle.dumps(file_list).split('\n'), file_list],
-            ]:
-
-            self.assertEquals(l,
-                MockLogProcessorDaemon(s).get_processed_files_list())
+        for s, l in [
+            ['', None],
+            [pickle.dumps(set()).split('\n'), set()],
+            [pickle.dumps(file_list).split('\n'), file_list],
+        ]:
+            self.assertEqual(
+                l, MockLogProcessorDaemon(s).get_processed_files_list())
 
     def test_get_processed_files_list_bad_file_downloads(self):
-        class MockLogProcessor():
+        class MockLogProcessor(object):
             def __init__(self, status_code):
                 self.err = log_common.BadFileDownload(status_code)
 
@@ -396,8 +401,8 @@ class TestLogProcessorDaemon(unittest.TestCase):
                 self.processed_files_filename = 'filename'
 
         for c, l in [[404, set()], [503, None], [None, None]]:
-            self.assertEquals(l,
-                MockLogProcessorDaemon(c).get_processed_files_list())
+            self.assertEqual(
+                l, MockLogProcessorDaemon(c).get_processed_files_list())
 
     def test_get_aggregate_data(self):
         # when run "for real"
@@ -414,7 +419,7 @@ class TestLogProcessorDaemon(unittest.TestCase):
                 'acct2_time1': {'field1': 6, 'field2': 7},
                 'acct3_time3': {'field1': 8, 'field2': 9},
                 }
-            ],
+             ],
             ['file2', {'acct1_time1': {'field1': 10}}],
         ]
 
@@ -433,9 +438,9 @@ class TestLogProcessorDaemon(unittest.TestCase):
         data_out = d.get_aggregate_data(processed_files, data_in)
 
         for k, v in expected_data_out.items():
-            self.assertEquals(v, data_out[k])
+            self.assertEqual(v, data_out[k])
 
-        self.assertEquals(set(['file1', 'file2']), processed_files)
+        self.assertEqual(set(['file1', 'file2']), processed_files)
 
     def test_get_final_info(self):
         # when run "for real"
@@ -457,7 +462,7 @@ class TestLogProcessorDaemon(unittest.TestCase):
 
         data_in = {
             'acct1_time1': {'field1': 11, 'field2': 2, 'field3': 3,
-                'field4': 8, 'field5': 11},
+                            'field4': 8, 'field5': 11},
             'acct1_time2': {'field1': 4, 'field2': 5},
             'acct2_time1': {'field1': 6, 'field2': 7},
             'acct3_time3': {'field1': 8, 'field2': 9},
@@ -465,43 +470,43 @@ class TestLogProcessorDaemon(unittest.TestCase):
 
         expected_data_out = {
             'acct1_time1': {'out_field1': 16, 'out_field2': 5,
-                'out_field3': 3, 'out_field4': 8, 'out_field5': 0,
-                'out_field6': 0, 'out_field7': 0},
+                            'out_field3': 3, 'out_field4': 8, 'out_field5': 0,
+                            'out_field6': 0, 'out_field7': 0},
             'acct1_time2': {'out_field1': 9, 'out_field2': 5,
-                'out_field3': 0, 'out_field4': 0, 'out_field5': 0,
-                'out_field6': 0, 'out_field7': 0},
+                            'out_field3': 0, 'out_field4': 0, 'out_field5': 0,
+                            'out_field6': 0, 'out_field7': 0},
             'acct2_time1': {'out_field1': 13, 'out_field2': 7,
-                'out_field3': 0, 'out_field4': 0, 'out_field5': 0,
-                'out_field6': 0, 'out_field7': 0},
+                            'out_field3': 0, 'out_field4': 0, 'out_field5': 0,
+                            'out_field6': 0, 'out_field7': 0},
             'acct3_time3': {'out_field1': 17, 'out_field2': 9,
-                'out_field3': 0, 'out_field4': 0, 'out_field5': 0,
-                'out_field6': 0, 'out_field7': 0},
+                            'out_field3': 0, 'out_field4': 0, 'out_field5': 0,
+                            'out_field6': 0, 'out_field7': 0},
         }
 
-        self.assertEquals(expected_data_out,
-            MockLogProcessorDaemon().get_final_info(data_in))
+        self.assertEqual(expected_data_out,
+                         MockLogProcessorDaemon().get_final_info(data_in))
 
     def test_store_processed_files_list(self):
-        class MockInternalProxy:
+        class MockInternalProxy(object):
             def __init__(self, test, daemon, processed_files):
                 self.test = test
                 self.daemon = daemon
                 self.processed_files = processed_files
 
             def upload_file(self, f, account, container, filename):
-                self.test.assertEquals(self.processed_files,
-                    pickle.loads(f.getvalue()))
-                self.test.assertEquals(self.daemon.log_processor_account,
-                    account)
-                self.test.assertEquals(self.daemon.log_processor_container,
-                    container)
-                self.test.assertEquals(self.daemon.processed_files_filename,
-                    filename)
+                self.test.assertEqual(self.processed_files,
+                                      pickle.loads(f.getvalue()))
+                self.test.assertEqual(self.daemon.log_processor_account,
+                                      account)
+                self.test.assertEqual(self.daemon.log_processor_container,
+                                      container)
+                self.test.assertEqual(self.daemon.processed_files_filename,
+                                      filename)
 
-        class MockLogProcessor:
+        class MockLogProcessor(object):
             def __init__(self, test, daemon, processed_files):
                 self.internal_proxy = MockInternalProxy(test, daemon,
-                    processed_files)
+                                                        processed_files)
 
         class MockLogProcessorDaemon(log_processor.LogProcessorDaemon):
             def __init__(self, test, processed_files):
@@ -537,13 +542,13 @@ class TestLogProcessorDaemon(unittest.TestCase):
         ]
 
         data_out = MockLogProcessorDaemon().get_output(data_in)
-        self.assertEquals(expected_data_out[0], data_out[0])
+        self.assertEqual(expected_data_out[0], data_out[0])
 
         for row in data_out[1:]:
-            self.assert_(row in expected_data_out)
+            self.assertTrue(row in expected_data_out)
 
         for row in expected_data_out[1:]:
-            self.assert_(row in data_out)
+            self.assertTrue(row in data_out)
 
     def test_store_output(self):
         try:
@@ -551,7 +556,7 @@ class TestLogProcessorDaemon(unittest.TestCase):
             mock_strftime_return = '2010/03/02/01/'
 
             def mock_strftime(format, t):
-                self.assertEquals('%Y/%m/%d/%H/', format)
+                self.assertEqual('%Y/%m/%d/%H/', format)
                 return mock_strftime_return
             log_processor.time.strftime = mock_strftime
 
@@ -567,32 +572,35 @@ class TestLogProcessorDaemon(unittest.TestCase):
             h = hashlib.md5(expected_output).hexdigest()
             expected_filename = '%s%s.csv.gz' % (mock_strftime_return, h)
 
-            class MockInternalProxy:
+            class MockInternalProxy(object):
                 def __init__(self, test, daemon, expected_filename,
-                    expected_output):
+                             expected_output):
                     self.test = test
                     self.daemon = daemon
                     self.expected_filename = expected_filename
                     self.expected_output = expected_output
 
                 def upload_file(self, f, account, container, filename):
-                    self.test.assertEquals(self.daemon.log_processor_account,
-                        account)
-                    self.test.assertEquals(self.daemon.log_processor_container,
-                        container)
-                    self.test.assertEquals(self.expected_filename, filename)
-                    self.test.assertEquals(self.expected_output, f.getvalue())
+                    self.test.assertEqual(self.daemon.log_processor_account,
+                                          account)
+                    self.test.assertEqual(self.daemon.log_processor_container,
+                                          container)
+                    self.test.assertEqual(self.expected_filename, filename)
+                    self.test.assertEqual(self.expected_output, f.getvalue())
 
-            class MockLogProcessor:
-                def __init__(self, test, daemon, expected_filename,
-                    expected_output):
-                    self.internal_proxy = MockInternalProxy(test, daemon,
-                        expected_filename, expected_output)
+            class MockLogProcessor(object):
+                def __init__(self, test,
+                             daemon,
+                             expected_filename,
+                             expected_output):
+                    self.internal_proxy = MockInternalProxy(
+                        test, daemon, expected_filename, expected_output)
 
             class MockLogProcessorDaemon(log_processor.LogProcessorDaemon):
                 def __init__(self, test, expected_filename, expected_output):
                     self.log_processor = MockLogProcessor(test, self,
-                        expected_filename, expected_output)
+                                                          expected_filename,
+                                                          expected_output)
                     self.log_processor_account = 'account'
                     self.log_processor_container = 'container'
                     self.processed_files_filename = 'filename'
@@ -612,7 +620,7 @@ class TestLogProcessorDaemon(unittest.TestCase):
 
         value_return = 'keylist_mapping'
 
-        class MockLogProcessor:
+        class MockLogProcessor(object):
             def __init__(self):
                 self.call_count = 0
 
@@ -626,16 +634,16 @@ class TestLogProcessorDaemon(unittest.TestCase):
                 self._keylist_mapping = None
 
         d = MockLogProcessorDaemon()
-        self.assertEquals(value_return, d.keylist_mapping)
-        self.assertEquals(value_return, d.keylist_mapping)
-        self.assertEquals(1, d.log_processor.call_count)
+        self.assertEqual(value_return, d.keylist_mapping)
+        self.assertEqual(value_return, d.keylist_mapping)
+        self.assertEqual(1, d.log_processor.call_count)
 
     def test_process_logs(self):
         try:
             mock_logs_to_process = 'logs_to_process'
             mock_processed_files = 'processed_files'
 
-            real_multiprocess_collate = log_processor.multiprocess_collate
+            real_multiprocess_collate = log_common.multiprocess_collate
             multiprocess_collate_return = 'multiprocess_collate_return'
 
             get_aggregate_data_return = 'get_aggregate_data_return'
@@ -650,19 +658,19 @@ class TestLogProcessorDaemon(unittest.TestCase):
                     self.worker_count = 'worker_count'
 
                 def get_aggregate_data(self, processed_files, results):
-                    self.test.assertEquals(mock_processed_files,
-                        processed_files)
-                    self.test.assertEquals(multiprocess_collate_return,
-                        results)
+                    self.test.assertEqual(mock_processed_files,
+                                          processed_files)
+                    self.test.assertEqual(multiprocess_collate_return,
+                                          results)
                     return get_aggregate_data_return
 
                 def get_final_info(self, aggr_data):
-                    self.test.assertEquals(get_aggregate_data_return,
-                        aggr_data)
+                    self.test.assertEqual(get_aggregate_data_return,
+                                          aggr_data)
                     return get_final_info_return
 
                 def get_output(self, final_info):
-                    self.test.assertEquals(get_final_info_return, final_info)
+                    self.test.assertEqual(get_final_info_return, final_info)
                     return get_output_return
 
             d = MockLogProcessorDaemon(self)
@@ -670,27 +678,27 @@ class TestLogProcessorDaemon(unittest.TestCase):
             def mock_multiprocess_collate(processor_klass, processor_args,
                                           processor_method, logs_to_process,
                                           worker_count):
-                self.assertEquals(d.total_conf, processor_args[0])
-                self.assertEquals(d.logger, processor_args[1])
+                self.assertEqual(d.total_conf, processor_args[0])
+                self.assertEqual(d.logger, processor_args[1])
 
-                self.assertEquals(mock_logs_to_process, logs_to_process)
-                self.assertEquals(d.worker_count, worker_count)
+                self.assertEqual(mock_logs_to_process, logs_to_process)
+                self.assertEqual(d.worker_count, worker_count)
 
                 return multiprocess_collate_return
 
-            log_processor.multiprocess_collate = mock_multiprocess_collate
+            log_common.multiprocess_collate = mock_multiprocess_collate
 
             output = d.process_logs(mock_logs_to_process, mock_processed_files)
-            self.assertEquals(get_output_return, output)
+            self.assertEqual(get_output_return, output)
         finally:
             log_processor.multiprocess_collate = real_multiprocess_collate
 
     def test_run_once_get_processed_files_list_returns_none(self):
-        class MockLogProcessor:
+        class MockLogProcessor(object):
             def get_data_list(self, lookback_start, lookback_end,
-                processed_files):
-                raise unittest.TestCase.failureException, \
-                    'Method should not be called'
+                              processed_files):
+                raise unittest.TestCase.failureException(
+                    'Method should not be called')
 
         class MockLogProcessorDaemon(log_processor.LogProcessorDaemon):
             def __init__(self):
@@ -706,22 +714,25 @@ class TestLogProcessorDaemon(unittest.TestCase):
         MockLogProcessorDaemon().run_once()
 
     def test_run_once_no_logs_to_process(self):
-        class MockLogProcessor():
+        class MockLogProcessor(object):
             def __init__(self, daemon, test):
                 self.daemon = daemon
                 self.test = test
 
-            def get_data_list(self, lookback_start, lookback_end,
-                processed_files):
-                self.test.assertEquals(self.daemon.lookback_start,
-                    lookback_start)
-                self.test.assertEquals(self.daemon.lookback_end,
-                    lookback_end)
-                self.test.assertEquals(self.daemon.processed_files,
-                    processed_files)
+            def get_data_list(self,
+                              lookback_start,
+                              lookback_end,
+                              processed_files):
+                self.test.assertEqual(self.daemon.lookback_start,
+                                      lookback_start)
+                self.test.assertEqual(self.daemon.lookback_end,
+                                      lookback_end)
+                self.test.assertEqual(self.daemon.processed_files,
+                                      processed_files)
                 return []
 
         class MockLogProcessorDaemon(log_processor.LogProcessorDaemon):
+
             def __init__(self, test):
                 self.logger = DumbLogger()
                 self.log_processor = MockLogProcessor(self, test)
@@ -736,8 +747,8 @@ class TestLogProcessorDaemon(unittest.TestCase):
                 return self.processed_files
 
             def process_logs(logs_to_process, processed_files):
-                raise unittest.TestCase.failureException, \
-                    'Method should not be called'
+                raise unittest.TestCase.failureException(
+                    'Method should not be called')
 
         MockLogProcessorDaemon(self).run_once()
 

test_slogging/unit/test_log_uploader.py

@@ -13,25 +13,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import unittest
-import os
 from datetime import datetime
-from tempfile import mkdtemp
-from shutil import rmtree
-from functools import partial
-from collections import defaultdict
-import random
-import string
-
-from test_slogging.unit import temptree
-from slogging import log_uploader
-
 import logging
+import os
+import random
+from slogging import log_uploader
+import string
+from test.unit import temptree
+import unittest
+
 logging.basicConfig(level=logging.DEBUG)
 LOGGER = logging.getLogger()
 
 COMPRESSED_DATA = '\x1f\x8b\x08\x08\x87\xa5zM\x02\xffdata\x00KI,I\x04\x00c' \
-        '\xf3\xf3\xad\x04\x00\x00\x00'
+                  '\xf3\xf3\xad\x04\x00\x00\x00'
+
+
+PROXY_SERVER_CONF = os.environ.get('SWIFT_PROXY_TEST_CONFIG_FILE',
+                                   '/etc/swift/proxy-server.conf')
 
 access_regex = '''
     ^
@@ -47,7 +46,7 @@ def mock_appconfig(*args, **kwargs):
     pass
 
 
-class MockInternalProxy():
+class MockInternalProxy(object):
 
     def __init__(self, *args, **kwargs):
         pass
@@ -105,11 +104,14 @@ class TestLogUploader(unittest.TestCase):
         with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t:
             # invalid pattern
             conf = {'log_dir': t,
+                    'proxy_server_conf': PROXY_SERVER_CONF,
                     'source_filename_pattern': '%Y%m%d%h'}  # should be %H
             uploader = MockLogUploader(conf)
             self.assertRaises(SystemExit, uploader.upload_all_logs)
 
-            conf = {'log_dir': t, 'source_filename_pattern': access_regex}
+            conf = {'log_dir': t,
+                    'proxy_server_conf': PROXY_SERVER_CONF,
+                    'source_filename_pattern': access_regex}
             uploader = ErrorLogUploader(conf)
             # this tests if the exception is handled
             uploader.upload_all_logs()
@@ -119,20 +121,24 @@ class TestLogUploader(unittest.TestCase):
         with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t:
             # invalid pattern
             conf = {'log_dir': t,
+                    'proxy_server_conf': PROXY_SERVER_CONF,
                     'source_filename_pattern': '%Y%m%d%h'}  # should be %H
             uploader = MockLogUploader(conf)
             self.assertRaises(SystemExit, uploader.upload_all_logs)
 
-            conf = {'log_dir': t, 'source_filename_pattern': access_regex}
+            conf = {'log_dir': t,
+                    'proxy_server_conf': PROXY_SERVER_CONF,
+                    'source_filename_pattern': access_regex}
             uploader = MockLogUploader(conf)
             uploader.upload_all_logs()
-            self.assertEquals(len(uploader.uploaded_files), 1)
+            self.assertEqual(len(uploader.uploaded_files), 1)
 
     def test_pattern_upload_all_logs(self):
 
         # test empty dir
         with temptree([]) as t:
-            conf = {'log_dir': t}
+            conf = {'log_dir': t,
+                    'proxy_server_conf': PROXY_SERVER_CONF}
             uploader = MockLogUploader(conf)
             self.assertRaises(SystemExit, uploader.run_once)
 
@@ -141,7 +147,7 @@ class TestLogUploader(unittest.TestCase):
                            range(random.randint(1, max_len)))
 
         template = 'prefix_%(random)s_%(digits)s.blah.' \
-                '%(datestr)s%(hour)0.2d00-%(next_hour)0.2d00-%(number)s.gz'
+                   '%(datestr)s%(hour)0.2d00-%(next_hour)0.2d00-%(number)s.gz'
         pattern = '''prefix_.*_[0-9]+\.blah\.
                      (?P<year>[0-9]{4})
                      (?P<month>[0-1][0-9])
@@ -175,42 +181,48 @@ class TestLogUploader(unittest.TestCase):
             files.append(fname)
 
         for fname in files:
-            print fname
+            print(fname)
 
         with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t:
-            self.assertEquals(len(os.listdir(t)), 48)
-            conf = {'source_filename_pattern': pattern, 'log_dir': t}
+            self.assertEqual(len(os.listdir(t)), 48)
+            conf = {'source_filename_pattern': pattern,
+                    'proxy_server_conf': PROXY_SERVER_CONF,
+                    'log_dir': t}
             uploader = MockLogUploader(conf)
             uploader.run_once()
-            self.assertEquals(len(os.listdir(t)), 24)
-            self.assertEquals(len(uploader.uploaded_files), 24)
+            self.assertEqual(len(os.listdir(t)), 24)
+            self.assertEqual(len(uploader.uploaded_files), 24)
             files_that_were_uploaded = set(x[0] for x in
                                            uploader.uploaded_files)
             for f in files_that_should_match:
-                self.assert_(os.path.join(t, f) in files_that_were_uploaded)
+                self.assertTrue(
+                    os.path.join(t, f) in files_that_were_uploaded)
 
     def test_log_cutoff(self):
         files = [datetime.now().strftime('%Y%m%d%H')]
         with temptree(files) as t:
             conf = {'log_dir': t, 'new_log_cutoff': '7200',
+                    'proxy_server_conf': PROXY_SERVER_CONF,
                     'source_filename_pattern': access_regex}
             uploader = MockLogUploader(conf)
             uploader.run_once()
-            self.assertEquals(len(uploader.uploaded_files), 0)
+            self.assertEqual(len(uploader.uploaded_files), 0)
             conf = {'log_dir': t, 'new_log_cutoff': '0',
+                    'proxy_server_conf': PROXY_SERVER_CONF,
                     'source_filename_pattern': access_regex}
             uploader = MockLogUploader(conf)
             uploader.run_once()
-            self.assertEquals(len(uploader.uploaded_files), 1)
+            self.assertEqual(len(uploader.uploaded_files), 1)
 
     def test_create_container_fail(self):
         files = [datetime.now().strftime('%Y%m%d%H')]
-        conf = {'source_filename_pattern': access_regex}
+        conf = {'source_filename_pattern': access_regex,
+                'proxy_server_conf': PROXY_SERVER_CONF}
         with temptree(files) as t:
             conf['log_dir'] = t
             uploader = MockLogUploader(conf)
             uploader.run_once()
-            self.assertEquals(len(uploader.uploaded_files), 1)
+            self.assertEqual(len(uploader.uploaded_files), 1)
 
         with temptree(files) as t:
             conf['log_dir'] = t
@@ -218,31 +230,34 @@ class TestLogUploader(unittest.TestCase):
             # mock create_container to fail
             uploader.internal_proxy.create_container = lambda *args: False
             uploader.run_once()
-            self.assertEquals(len(uploader.uploaded_files), 0)
+            self.assertEqual(len(uploader.uploaded_files), 0)
 
     def test_unlink_log(self):
         files = [datetime.now().strftime('%Y%m%d%H')]
         with temptree(files, contents=[COMPRESSED_DATA]) as t:
             conf = {'log_dir': t, 'unlink_log': 'false',
+                    'proxy_server_conf': PROXY_SERVER_CONF,
                     'source_filename_pattern': access_regex}
             uploader = MockLogUploader(conf)
             uploader.run_once()
-            self.assertEquals(len(uploader.uploaded_files), 1)
+            self.assertEqual(len(uploader.uploaded_files), 1)
             # file still there
-            self.assertEquals(len(os.listdir(t)), 1)
+            self.assertEqual(len(os.listdir(t)), 1)
 
             conf = {'log_dir': t, 'unlink_log': 'true',
+                    'proxy_server_conf': PROXY_SERVER_CONF,
                     'source_filename_pattern': access_regex}
             uploader = MockLogUploader(conf)
             uploader.run_once()
-            self.assertEquals(len(uploader.uploaded_files), 1)
+            self.assertEqual(len(uploader.uploaded_files), 1)
             # file gone
-            self.assertEquals(len(os.listdir(t)), 0)
+            self.assertEqual(len(os.listdir(t)), 0)
 
     def test_upload_file_failed(self):
         files = ['plugin-%s' % datetime.now().strftime('%Y%m%d%H')]
         with temptree(files, contents=[COMPRESSED_DATA]) as t:
             conf = {'log_dir': t, 'unlink_log': 'true',
+                    'proxy_server_conf': PROXY_SERVER_CONF,
                     'source_filename_pattern': access_regex}
             uploader = MockLogUploader(conf)
 
@@ -253,7 +268,7 @@ class TestLogUploader(unittest.TestCase):
             uploader.internal_proxy.upload_file = mock_upload_file
             self.assertRaises(SystemExit, uploader.run_once)
             # file still there
-            self.assertEquals(len(os.listdir(t)), 1)
+            self.assertEqual(len(os.listdir(t)), 1)
 
 
 if __name__ == '__main__':

test_slogging/unit/test_stats_processor.py

@@ -13,8 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import unittest
 from slogging import stats_processor
+import unittest
 
 
 class TestStatsProcessor(unittest.TestCase):
@@ -23,28 +23,29 @@ class TestStatsProcessor(unittest.TestCase):
         p = stats_processor.StatsLogProcessor({})
         test_obj_stream = ['a, 1, 1, 1', 'a, 1, 1, 1']
         res = p.process(test_obj_stream, 'foo', 'bar', '2011/03/14/12/baz')
-        expected = {('a', '2011', '03', '14', '12'):
-                        {'object_count': 2, 'container_count': 2,
-                        'replica_count': 2, 'bytes_used': 2}}
-        self.assertEquals(res, expected)
+        expected = {('a', '2011', '03', '14', '12'): {
+            'object_count': 2, 'container_count': 2,
+            'replica_count': 2, 'bytes_used': 2}}
+        self.assertEqual(res, expected)
 
     def test_process_extra_columns(self):
         p = stats_processor.StatsLogProcessor({})
         test_obj_stream = ['a, 1, 1, 1, extra']
         res = p.process(test_obj_stream, 'foo', 'bar', '2011/03/14/12/baz')
-        expected = {('a', '2011', '03', '14', '12'):
-                        {'object_count': 1, 'container_count': 1,
-                        'replica_count': 1, 'bytes_used': 1}}
-        self.assertEquals(res, expected)
+        expected = {('a', '2011', '03', '14', '12'): {
+            'object_count': 1, 'container_count': 1,
+            'replica_count': 1, 'bytes_used': 1}}
+        self.assertEqual(res, expected)
 
     def test_process_bad_line(self):
         p = stats_processor.StatsLogProcessor({})
         test_obj_stream = ['a, 1, 1, 1, extra', '', 'some bad line']
         res = p.process(test_obj_stream, 'foo', 'bar', '2011/03/14/12/baz')
-        expected = {('a', '2011', '03', '14', '12'):
-                        {'object_count': 1, 'container_count': 1,
-                        'replica_count': 1, 'bytes_used': 1}}
-        self.assertEquals(res, expected)
+        expected = {
+            ('a', '2011', '03', '14', '12'): {
+                'object_count': 1, 'container_count': 1,
+                'replica_count': 1, 'bytes_used': 1}}
+        self.assertEqual(res, expected)
 
 
 if __name__ == '__main__':

tools/pip-requires

@@ -1 +0,0 @@
-iptools>=0.4

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py26,pep8
+envlist = py27,docs,pep8
 
 [testenv]
 setenv = VIRTUAL_ENV={envdir}
@@ -9,24 +9,66 @@ setenv = VIRTUAL_ENV={envdir}
          NOSE_OPENSTACK_YELLOW=0.025
          NOSE_OPENSTACK_SHOW_ELAPSED=1
          NOSE_OPENSTACK_STDOUT=1
-deps =
-  {toxinidir}/../swift
-  -r{toxinidir}/../swift/requirements.txt
-  -r{toxinidir}/../swift/test-requirements.txt
-  -r{toxinidir}/tools/pip-requires
-commands = nosetests {posargs}
+deps = -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt}
+commands=
+  py27: {[unit_tests]commands}
+  py35: {[unit_tests]commands}
 
-[tox:jenkins]
-downloadcache = ~/cache/pip
 
 [testenv:pep8]
-deps = pep8==1.1
-commands =
-  pep8 --repeat --show-pep8 --show-source --ignore=W602 \
-   --exclude=.venv,.tox,dist,doc,test .
+deps = http://tarballs.openstack.org/swift/swift-stable-queens.tar.gz
+commands = flake8 {posargs}
+
 
 [testenv:cover]
-setenv = NOSE_WITH_COVERAGE=1
+setenv = VIRTUAL_ENV={envdir}
+         NOSE_WITH_COVERAGE=1
+         NOSE_COVER_HTML_DIR={toxinidir}/cover
+         NOSE_COVER_HTML=1
+         NOSE_COVER_ERASE=1
+commands =
+  {[unit_tests]commands}
+
+
+[testenv:py27]
+basepython = python2.7
+deps = http://tarballs.openstack.org/swift/swift-stable-queens.tar.gz
+setenv = SWIFT_TEST_CONFIG_FILE={toxinidir}/test/sample.conf
+         SWIFT_PROXY_TEST_CONFIG_FILE={toxinidir}/test/sample.proxy-server.conf
+commands =
+  {[unit_tests]commands}
+
+
+[unit_tests]
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+commands = find . -type f -name "*.py[c|o]" -delete
+           find . -type d -name "__pycache__" -delete
+           nosetests {posargs:test/unit}
+
 
 [testenv:venv]
 commands = {posargs}
+
+[flake8]
+exclude = .venv,.git,.tox,dist,*lib/python*,*egg,build,dash_template
+max-complexity = 20
+import-order-style = pep8
+
+
+[testenv:docs]
+deps = http://tarballs.openstack.org/swift/swift-stable-queens.tar.gz
+       -r{toxinidir}/doc/requirements.txt
+commands = python setup.py build_sphinx
+
+
+[doc8]
+# File extensions to check
+extensions = .rst, .yaml
+# Maximal line length should be 80 but we have some overlong lines.
+# Let's not get far more in.
+max-line-length = 80
+# Disable some doc8 checks:
+# D000: Check RST validity
+# - cannot handle "none" for code-block directive
+ignore = D000