Test role for upload-logs-swift

This creates a testing role for upload-logs-swift so that we can test
chagnes to this role before moving them into the production role.

Change-Id: If55b0e2809d9309bc98e6763da32219d996dbfa4

doc/source/deprecated-roles.rst

@@ -3,3 +3,4 @@ Deprecrated and Test Roles
 
 .. zuul:autorole:: fetch-zuul-cloner
 .. zuul:autorole:: test-mirror-workspace-git-repos
+.. zuul:autorole:: test-upload-logs-swift

roles/test-upload-logs-swift/README.rst

@@ -0,0 +1,80 @@
+THIS IS FOR TESTING ONLY
+
+Upload logs to a swift container
+
+This uploads logs to an OpenStack Object Store (Swift) container.
+
+**Role Variables**
+
+.. zuul:rolevar:: zuul_site_upload_logs
+   :default: true
+
+   Controls when logs are uploaded. true, the default, means always
+   upload logs. false means never upload logs. 'failure' means to only
+   upload logs when the job has failed.
+
+   .. note:: Intended to be set by admins via site-variables.
+
+.. zuul:rolevar:: zuul_log_cloud_config
+
+   Complex argument which contains the cloud configuration in
+   os-cloud-config (clouds.yaml) format.  It is expected that this
+   argument comes from a `Secret`.
+
+.. zuul:rolevar:: zuul_log_partition
+   :default: false
+
+   If set to true, then the first component of the log path will be
+   removed from the object name and added to the container name, so
+   that logs for different changes are distributed across a large
+   number of containers.
+
+.. zuul:rolevar:: zuul_log_container
+   :default: logs
+
+   This role will create containers which do not already exist.  If
+   partitioning is not enabled, this is the name of the container
+   which will be used.  If partitioning is enabled, then this will be
+   used as the prefix for the container name which will be separated
+   from the partition name by an underscore.  For example, "logs_42"
+   would be the container name for partition 42.
+
+   Note that you will want to set this to a value that uniquely
+   identifies your Zuul installation if using shared object stores that
+   require globally unique container names. For example if using a
+   public cloud whose Swift API is provided by Ceph.
+
+.. zuul:rolevar:: zuul_log_container_public
+   :default: true
+
+   If the container is created, should it be created with global read
+   ACLs.  If the container already exists, it will not be modified.
+
+.. zuul:rolevar:: zuul_log_delete_after
+   :default: 15552000
+
+   Number of seconds to delete objects after upload. Default is 6 months
+   (15552000 seconds) and if set to 0 X-Delete-After will not be set.
+
+.. zuul:rolevar:: zuul_log_path
+   :default: Generated by the role `set-zuul-log-path-fact`
+
+   Prepend this path to the object names when uploading.
+
+.. zuul:rolevar:: zuul_log_create_indexes
+   :default: true
+
+   Whether to create `index.html` files with directory indexes.  If set
+   to false, Swift containers can be marked with a `Web-Listings=true`
+   property to activate Swift's own directory indexing.
+
+.. zuul:rolevar:: zuul_log_path_shard_build
+   :default: False
+
+   This var is consumed by set-zuul-log-path-fact which upload-logs-swift
+   calls into. If you set this you will get log paths prefixed with the
+   first three characters of the build uuid. This will improve log file
+   sharding.
+
+   More details can be found at
+   :zuul:rolevar:`set-zuul-log-path-fact.zuul_log_path_shard_build`.

roles/test-upload-logs-swift/defaults/main.yaml

@@ -0,0 +1,4 @@
+zuul_log_partition: false
+zuul_log_container: logs
+zuul_log_container_public: true
+zuul_log_create_indexes: true

roles/test-upload-logs-swift/library/delete_container.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+#
+# Copyright 2019 Red Hat, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import openstack
+import requests
+import logging
+import os
+
+logging.basicConfig(level=logging.INFO)
+# logging.getLogger("requests").setLevel(logging.DEBUG)
+# logging.getLogger("keystoneauth").setLevel(logging.INFO)
+# logging.getLogger("stevedore").setLevel(logging.INFO)
+logging.captureWarnings(True)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Delete a swift container"
+    )
+    parser.add_argument('cloud',
+                        help='Name of the cloud to use when uploading')
+    parser.add_argument('container',
+                        help='Name of the container to use when uploading')
+
+    args = parser.parse_args()
+
+    cloud = openstack.connect(cloud=args.cloud)
+
+    sess = cloud.config.get_session()
+    adapter = requests.adapters.HTTPAdapter(pool_maxsize=100)
+    sess.mount('https://', adapter)
+
+    container = cloud.get_container(args.container)
+    print('Found container', container)
+    print()
+    for x in cloud.object_store.objects(args.container):
+        print('Delete object', x.name)
+        if x.name == '/':
+            endpoint = cloud.object_store.get_endpoint()
+            container = os.path.join(endpoint, args.container)
+            cloud.session.delete(container + '//')
+        else:
+            cloud.object_store.delete_object(x)
+
+    print()
+    print('Delete container', container)
+    cloud.object_store.delete_container(args.container)
+
+
+if __name__ == "__main__":
+    main()

roles/test-upload-logs-swift/library/test-fixtures/links/job-output.json

@@ -0,0 +1 @@
+{"test": "foo"}

roles/test-upload-logs-swift/library/test-fixtures/logs/controller/cpu-load.svg

@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg>
+</svg>

roles/test-upload-logs-swift/library/test-fixtures/logs/controller/subdir/foo::3.txt

@@ -0,0 +1,2 @@
+This is a plan text file with a funny name.
+The index links should escape the :'s.

roles/test-upload-logs-swift/library/test-fixtures/logs/job-output.json

@@ -0,0 +1 @@
+{"test": "foo"}

roles/test-upload-logs-swift/library/test_zuul_swift_upload.py

@@ -0,0 +1,404 @@
+# Copyright (C) 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import testtools
+import time
+import stat
+import fixtures
+
+from bs4 import BeautifulSoup
+from .zuul_swift_upload import FileList, Indexer, FileDetail
+
+
+FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
+                           'test-fixtures')
+
+
+class SymlinkFixture(fixtures.Fixture):
+    links = [
+        ('bad_symlink', '/etc'),
+        ('bad_symlink_file', '/etc/issue'),
+        ('good_symlink', 'controller'),
+        ('recursive_symlink', '.'),
+        ('symlink_file', 'job-output.json'),
+        ('symlink_loop_a', 'symlink_loop'),
+        ('symlink_loop/symlink_loop_b', '..'),
+    ]
+
+    def _setUp(self):
+        self._cleanup()
+        for (src, target) in self.links:
+            path = os.path.join(FIXTURE_DIR, 'links', src)
+            os.symlink(target, path)
+        self.addCleanup(self._cleanup)
+
+    def _cleanup(self):
+        for (src, target) in self.links:
+            path = os.path.join(FIXTURE_DIR, 'links', src)
+            if os.path.exists(path):
+                os.unlink(path)
+
+
+class TestFileList(testtools.TestCase):
+
+    def assert_files(self, result, files):
+        self.assertEqual(len(result), len(files))
+        for expected, received in zip(files, result):
+            self.assertEqual(expected[0], received.relative_path)
+            if expected[0] and expected[0][-1] == '/':
+                efilename = os.path.split(
+                    os.path.dirname(expected[0]))[1] + '/'
+            else:
+                efilename = os.path.split(expected[0])[1]
+            self.assertEqual(efilename, received.filename)
+            if received.folder:
+                if received.full_path is not None and expected[0] != '':
+                    self.assertTrue(os.path.isdir(received.full_path))
+            else:
+                self.assertTrue(os.path.isfile(received.full_path))
+            self.assertEqual(expected[1], received.mimetype)
+            self.assertEqual(expected[2], received.encoding)
+
+    def find_file(self, file_list, path):
+        for f in file_list:
+            if f.relative_path == path:
+                return f
+
+    def test_single_dir_trailing_slash(self):
+        '''Test a single directory with a trailing slash'''
+
+        with FileList() as fl:
+            fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
+            self.assert_files(fl, [
+                ('', 'application/directory', None),
+                ('controller', 'application/directory', None),
+                ('zuul-info', 'application/directory', None),
+                ('job-output.json', 'application/json', None),
+                ('controller/subdir', 'application/directory', None),
+                ('controller/compressed.gz', 'text/plain', 'gzip'),
+                ('controller/cpu-load.svg', 'image/svg+xml', None),
+                ('controller/journal.xz', 'text/plain', 'xz'),
+                ('controller/service_log.txt', 'text/plain', None),
+                ('controller/syslog', 'text/plain', None),
+                ('controller/subdir/foo::3.txt', 'text/plain', None),
+                ('controller/subdir/subdir.txt', 'text/plain', None),
+                ('zuul-info/inventory.yaml', 'text/plain', None),
+                ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
+            ])
+
+    def test_single_dir(self):
+        '''Test a single directory without a trailing slash'''
+        with FileList() as fl:
+            fl.add(os.path.join(FIXTURE_DIR, 'logs'))
+            self.assert_files(fl, [
+                ('', 'application/directory', None),
+                ('logs', 'application/directory', None),
+                ('logs/controller', 'application/directory', None),
+                ('logs/zuul-info', 'application/directory', None),
+                ('logs/job-output.json', 'application/json', None),
+                ('logs/controller/subdir', 'application/directory', None),
+                ('logs/controller/compressed.gz', 'text/plain', 'gzip'),
+                ('logs/controller/cpu-load.svg', 'image/svg+xml', None),
+                ('logs/controller/journal.xz', 'text/plain', 'xz'),
+                ('logs/controller/service_log.txt', 'text/plain', None),
+                ('logs/controller/syslog', 'text/plain', None),
+                ('logs/controller/subdir/foo::3.txt', 'text/plain', None),
+                ('logs/controller/subdir/subdir.txt', 'text/plain', None),
+                ('logs/zuul-info/inventory.yaml', 'text/plain', None),
+                ('logs/zuul-info/zuul-info.controller.txt',
+                 'text/plain', None),
+            ])
+
+    def test_single_file(self):
+        '''Test a single file'''
+        with FileList() as fl:
+            fl.add(os.path.join(FIXTURE_DIR,
+                                'logs/zuul-info/inventory.yaml'))
+            self.assert_files(fl, [
+                ('', 'application/directory', None),
+                ('inventory.yaml', 'text/plain', None),
+            ])
+
+    def test_symlinks(self):
+        '''Test symlinks'''
+        with FileList() as fl:
+            self.useFixture(SymlinkFixture())
+            fl.add(os.path.join(FIXTURE_DIR, 'links/'))
+            self.assert_files(fl, [
+                ('', 'application/directory', None),
+                ('controller', 'application/directory', None),
+                ('good_symlink', 'application/directory', None),
+                ('recursive_symlink', 'application/directory', None),
+                ('symlink_loop', 'application/directory', None),
+                ('symlink_loop_a', 'application/directory', None),
+                ('job-output.json', 'application/json', None),
+                ('symlink_file', 'text/plain', None),
+                ('controller/service_log.txt', 'text/plain', None),
+                ('symlink_loop/symlink_loop_b', 'application/directory', None),
+                ('symlink_loop/placeholder', 'text/plain', None),
+            ])
+
+    def test_index_files(self):
+        '''Test index generation'''
+        with FileList() as fl:
+            fl.add(os.path.join(FIXTURE_DIR, 'logs'))
+            ix = Indexer(fl)
+            ix.make_indexes()
+
+            self.assert_files(fl, [
+                ('', 'application/directory', None),
+                ('index.html', 'text/html', None),
+                ('logs', 'application/directory', None),
+                ('logs/controller', 'application/directory', None),
+                ('logs/zuul-info', 'application/directory', None),
+                ('logs/job-output.json', 'application/json', None),
+                ('logs/index.html', 'text/html', None),
+                ('logs/controller/subdir', 'application/directory', None),
+                ('logs/controller/compressed.gz', 'text/plain', 'gzip'),
+                ('logs/controller/cpu-load.svg', 'image/svg+xml', None),
+                ('logs/controller/journal.xz', 'text/plain', 'xz'),
+                ('logs/controller/service_log.txt', 'text/plain', None),
+                ('logs/controller/syslog', 'text/plain', None),
+                ('logs/controller/index.html', 'text/html', None),
+                ('logs/controller/subdir/foo::3.txt', 'text/plain', None),
+                ('logs/controller/subdir/subdir.txt', 'text/plain', None),
+                ('logs/controller/subdir/index.html', 'text/html', None),
+                ('logs/zuul-info/inventory.yaml', 'text/plain', None),
+                ('logs/zuul-info/zuul-info.controller.txt',
+                 'text/plain', None),
+                ('logs/zuul-info/index.html', 'text/html', None),
+            ])
+
+            top_index = self.find_file(fl, 'index.html')
+            page = open(top_index.full_path).read()
+            page = BeautifulSoup(page, 'html.parser')
+            rows = page.find_all('tr')[1:]
+
+            self.assertEqual(len(rows), 1)
+
+            self.assertEqual(rows[0].find('a').get('href'), 'logs/')
+            self.assertEqual(rows[0].find('a').text, 'logs/')
+
+            subdir_index = self.find_file(
+                fl, 'logs/controller/subdir/index.html')
+            page = open(subdir_index.full_path).read()
+            page = BeautifulSoup(page, 'html.parser')
+            rows = page.find_all('tr')[1:]
+            self.assertEqual(rows[0].find('a').get('href'), '../')
+            self.assertEqual(rows[0].find('a').text, '../')
+
+            # Test proper escaping of files with funny names
+            self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt')
+            self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
+            # Test files without escaping
+            self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
+            self.assertEqual(rows[2].find('a').text, 'subdir.txt')
+
+    def test_index_files_trailing_slash(self):
+        '''Test index generation with a trailing slash'''
+        with FileList() as fl:
+            fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
+            ix = Indexer(fl)
+            ix.make_indexes()
+
+            self.assert_files(fl, [
+                ('', 'application/directory', None),
+                ('controller', 'application/directory', None),
+                ('zuul-info', 'application/directory', None),
+                ('job-output.json', 'application/json', None),
+                ('index.html', 'text/html', None),
+                ('controller/subdir', 'application/directory', None),
+                ('controller/compressed.gz', 'text/plain', 'gzip'),
+                ('controller/cpu-load.svg', 'image/svg+xml', None),
+                ('controller/journal.xz', 'text/plain', 'xz'),
+                ('controller/service_log.txt', 'text/plain', None),
+                ('controller/syslog', 'text/plain', None),
+                ('controller/index.html', 'text/html', None),
+                ('controller/subdir/foo::3.txt', 'text/plain', None),
+                ('controller/subdir/subdir.txt', 'text/plain', None),
+                ('controller/subdir/index.html', 'text/html', None),
+                ('zuul-info/inventory.yaml', 'text/plain', None),
+                ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
+                ('zuul-info/index.html', 'text/html', None),
+            ])
+
+            top_index = self.find_file(fl, 'index.html')
+            page = open(top_index.full_path).read()
+            page = BeautifulSoup(page, 'html.parser')
+            rows = page.find_all('tr')[1:]
+
+            self.assertEqual(len(rows), 3)
+
+            self.assertEqual(rows[0].find('a').get('href'), 'controller/')
+            self.assertEqual(rows[0].find('a').text, 'controller/')
+
+            self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
+            self.assertEqual(rows[1].find('a').text, 'zuul-info/')
+
+            subdir_index = self.find_file(fl, 'controller/subdir/index.html')
+            page = open(subdir_index.full_path).read()
+            page = BeautifulSoup(page, 'html.parser')
+            rows = page.find_all('tr')[1:]
+            self.assertEqual(rows[0].find('a').get('href'), '../')
+            self.assertEqual(rows[0].find('a').text, '../')
+
+            # Test proper escaping of files with funny names
+            self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt')
+            self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
+            # Test files without escaping
+            self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
+            self.assertEqual(rows[2].find('a').text, 'subdir.txt')
+
+    def test_topdir_parent_link(self):
+        '''Test index generation creates topdir parent link'''
+        with FileList() as fl:
+            fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
+            ix = Indexer(fl)
+            ix.make_indexes(
+                create_parent_links=True,
+                create_topdir_parent_link=True)
+
+            self.assert_files(fl, [
+                ('', 'application/directory', None),
+                ('controller', 'application/directory', None),
+                ('zuul-info', 'application/directory', None),
+                ('job-output.json', 'application/json', None),
+                ('index.html', 'text/html', None),
+                ('controller/subdir', 'application/directory', None),
+                ('controller/compressed.gz', 'text/plain', 'gzip'),
+                ('controller/cpu-load.svg', 'image/svg+xml', None),
+                ('controller/journal.xz', 'text/plain', 'xz'),
+                ('controller/service_log.txt', 'text/plain', None),
+                ('controller/syslog', 'text/plain', None),
+                ('controller/index.html', 'text/html', None),
+                ('controller/subdir/foo::3.txt', 'text/plain', None),
+                ('controller/subdir/subdir.txt', 'text/plain', None),
+                ('controller/subdir/index.html', 'text/html', None),
+                ('zuul-info/inventory.yaml', 'text/plain', None),
+                ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
+                ('zuul-info/index.html', 'text/html', None),
+            ])
+
+            top_index = self.find_file(fl, 'index.html')
+            page = open(top_index.full_path).read()
+            page = BeautifulSoup(page, 'html.parser')
+            rows = page.find_all('tr')[1:]
+
+            self.assertEqual(len(rows), 4)
+
+            self.assertEqual(rows[0].find('a').get('href'), '../')
+            self.assertEqual(rows[0].find('a').text, '../')
+
+            self.assertEqual(rows[1].find('a').get('href'), 'controller/')
+            self.assertEqual(rows[1].find('a').text, 'controller/')
+
+            self.assertEqual(rows[2].find('a').get('href'), 'zuul-info/')
+            self.assertEqual(rows[2].find('a').text, 'zuul-info/')
+
+            subdir_index = self.find_file(fl, 'controller/subdir/index.html')
+            page = open(subdir_index.full_path).read()
+            page = BeautifulSoup(page, 'html.parser')
+            rows = page.find_all('tr')[1:]
+            self.assertEqual(rows[0].find('a').get('href'), '../')
+            self.assertEqual(rows[0].find('a').text, '../')
+
+            # Test proper escaping of files with funny names
+            self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt')
+            self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
+            # Test files without escaping
+            self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
+            self.assertEqual(rows[2].find('a').text, 'subdir.txt')
+
+    def test_no_parent_links(self):
+        '''Test index generation creates topdir parent link'''
+        with FileList() as fl:
+            fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
+            ix = Indexer(fl)
+            ix.make_indexes(
+                create_parent_links=False,
+                create_topdir_parent_link=False)
+
+            self.assert_files(fl, [
+                ('', 'application/directory', None),
+                ('controller', 'application/directory', None),
+                ('zuul-info', 'application/directory', None),
+                ('job-output.json', 'application/json', None),
+                ('index.html', 'text/html', None),
+                ('controller/subdir', 'application/directory', None),
+                ('controller/compressed.gz', 'text/plain', 'gzip'),
+                ('controller/cpu-load.svg', 'image/svg+xml', None),
+                ('controller/journal.xz', 'text/plain', 'xz'),
+                ('controller/service_log.txt', 'text/plain', None),
+                ('controller/syslog', 'text/plain', None),
+                ('controller/index.html', 'text/html', None),
+                ('controller/subdir/foo::3.txt', 'text/plain', None),
+                ('controller/subdir/subdir.txt', 'text/plain', None),
+                ('controller/subdir/index.html', 'text/html', None),
+                ('zuul-info/inventory.yaml', 'text/plain', None),
+                ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
+                ('zuul-info/index.html', 'text/html', None),
+            ])
+
+            top_index = self.find_file(fl, 'index.html')
+            page = open(top_index.full_path).read()
+            page = BeautifulSoup(page, 'html.parser')
+            rows = page.find_all('tr')[1:]
+
+            self.assertEqual(len(rows), 3)
+
+            self.assertEqual(rows[0].find('a').get('href'), 'controller/')
+            self.assertEqual(rows[0].find('a').text, 'controller/')
+
+            self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
+            self.assertEqual(rows[1].find('a').text, 'zuul-info/')
+
+            subdir_index = self.find_file(fl, 'controller/subdir/index.html')
+            page = open(subdir_index.full_path).read()
+            page = BeautifulSoup(page, 'html.parser')
+            rows = page.find_all('tr')[1:]
+
+            # Test proper escaping of files with funny names
+            self.assertEqual(rows[0].find('a').get('href'), 'foo%3A%3A3.txt')
+            self.assertEqual(rows[0].find('a').text, 'foo::3.txt')
+            # Test files without escaping
+            self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
+            self.assertEqual(rows[1].find('a').text, 'subdir.txt')
+
+
+class TestFileDetail(testtools.TestCase):
+
+    def test_get_file_detail(self):
+        '''Test files info'''
+        path = os.path.join(FIXTURE_DIR, 'logs/job-output.json')
+        file_detail = FileDetail(path, '')
+        path_stat = os.stat(path)
+        self.assertEqual(
+            time.gmtime(path_stat[stat.ST_MTIME]),
+            file_detail.last_modified)
+        self.assertEqual(16, file_detail.size)
+
+    def test_get_file_detail_missing_file(self):
+        '''Test files that go missing during a walk'''
+
+        file_detail = FileDetail('missing/file/that/we/cant/find', '')
+
+        self.assertEqual(time.gmtime(0), file_detail.last_modified)
+        self.assertEqual(0, file_detail.size)

roles/test-upload-logs-swift/library/zuul_swift_upload.py

@@ -0,0 +1,915 @@
+#!/usr/bin/env python3
+#
+# Copyright 2014 Rackspace Australia
+# Copyright 2018 Red Hat, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+"""
+Utility to upload files to swift
+"""
+
+import argparse
+import gzip
+import io
+import logging
+import mimetypes
+import os
+try:
+    import queue as queuelib
+except ImportError:
+    import Queue as queuelib
+import shutil
+import stat
+import sys
+import tempfile
+import threading
+import time
+import traceback
+try:
+    import urllib.parse as urlparse
+except ImportError:
+    import urllib as urlparse
+import zlib
+import collections
+
+import openstack
+import requests
+import requests.exceptions
+import requestsexceptions
+import keystoneauth1.exceptions
+
+from ansible.module_utils.basic import AnsibleModule
+
+try:
+    # Python 3.3+
+    from collections.abc import Sequence
+except ImportError:
+    from collections import Sequence
+
+mimetypes.init()
+mimetypes.add_type('text/plain', '.yaml')
+
+MAX_UPLOAD_THREADS = 24
+POST_ATTEMPTS = 3
+
+# Map mime types to apache icons
+APACHE_MIME_ICON_MAP = {
+    '_default': 'unknown.png',
+    'application/gzip': 'compressed.png',
+    'application/directory': 'folder.png',
+    'text/html': 'text.png',
+    'text/plain': 'text.png',
+}
+
+# Map mime types to apache icons
+APACHE_FILE_ICON_MAP = {
+    '..': 'back.png',
+}
+
+# These icon files are from the Apache project and are in the public
+# domain.
+ICON_IMAGES = {
+    'back.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAElBMVEX/'
+                '///M//+ZmZlmZmYzMzMAAACei5rnAAAAAnRSTlP/AOW3MEoAAABWSURB'
+                'VHjabdBBCgAhDEPRRpv7X3kwEMsQ//IRRC08urjRHbha5VLFUsVSxVI9'
+                'lmDh5hMpHD6n0EgoiZG0DNINpnWlcVXaRix76e1/8dddcL6nG0Ri9gHj'
+                'tgSXKYeLBgAAAABJRU5ErkJggg==',
+    'compressed.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAADAFBM'
+                      'VEX//////8z//5n//2b//zP//wD/zP//zMz/zJn/zGb/zDP/zAD/'
+                      'mf//mcz/mZn/mWb/mTP/mQD/Zv//Zsz/Zpn/Zmb/ZjP/ZgD/M///'
+                      'M8z/M5n/M2b/MzP/MwD/AP//AMz/AJn/AGb/ADP/AADM///M/8zM'
+                      '/5nM/2bM/zPM/wDMzP/MzMzMzJnMzGbMzDPMzADMmf/MmczMmZnM'
+                      'mWbMmTPMmQDMZv/MZszMZpnMZmbMZjPMZgDMM//MM8zMM5nMM2bM'
+                      'MzPMMwDMAP/MAMzMAJnMAGbMADPMAACZ//+Z/8yZ/5mZ/2aZ/zOZ'
+                      '/wCZzP+ZzMyZzJmZzGaZzDOZzACZmf+ZmcyZmZmZmWaZmTOZmQCZ'
+                      'Zv+ZZsyZZpmZZmaZZjOZZgCZM/+ZM8yZM5mZM2aZMzOZMwCZAP+Z'
+                      'AMyZAJmZAGaZADOZAABm//9m/8xm/5lm/2Zm/zNm/wBmzP9mzMxm'
+                      'zJlmzGZmzDNmzABmmf9mmcxmmZlmmWZmmTNmmQBmZv9mZsxmZplm'
+                      'ZmZmZjNmZgBmM/9mM8xmM5lmM2ZmMzNmMwBmAP9mAMxmAJlmAGZm'
+                      'ADNmAAAz//8z/8wz/5kz/2Yz/zMz/wAzzP8zzMwzzJkzzGYzzDMz'
+                      'zAAzmf8zmcwzmZkzmWYzmTMzmQAzZv8zZswzZpkzZmYzZjMzZgAz'
+                      'M/8zM8wzM5kzM2YzMzMzMwAzAP8zAMwzAJkzAGYzADMzAAAA//8A'
+                      '/8wA/5kA/2YA/zMA/wAAzP8AzMwAzJkAzGYAzDMAzAAAmf8AmcwA'
+                      'mZkAmWYAmTMAmQAAZv8AZswAZpkAZmYAZjMAZgAAM/8AM8wAM5kA'
+                      'M2YAMzMAMwAAAP8AAMwAAJkAAGYAADPuAADdAAC7AACqAACIAAB3'
+                      'AABVAABEAAAiAAARAAAA7gAA3QAAuwAAqgAAiAAAdwAAVQAARAAA'
+                      'IgAAEQAAAO4AAN0AALsAAKoAAIgAAHcAAFUAAEQAACIAABHu7u7d'
+                      '3d27u7uqqqqIiIh3d3dVVVVEREQiIiIREREAAAD7CIKZAAAAJXRS'
+                      'TlP///////////////////////////////////////////////8A'
+                      'P89CTwAAAGtJREFUeNp9z9ENgDAIhOEOco+dybVuEXasFMRDY/x5'
+                      '+xJCO6Znu6kSx7BhXyjtKBWWNlwW88Loid7hFRKBXiIYCMfMEYUQ'
+                      'QohC3CjFA5nIjqx1CqlDLGR/EhM5O06yvin0ftGOyIS7lV14AsQN'
+                      'aR7rMEBYAAAAAElFTkSuQmCC',
+    'folder.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAElBMVEX/'
+                  '////zJnM//+ZZjMzMzMAAADCEvqoAAAAA3RSTlP//wDXyg1BAAAASElE'
+                  'QVR42s3KQQ6AQAhDUaXt/a/sQDrRJu7c+NmQB0e99B3lnqjT6cYx6zSI'
+                  'bV40n3D7psYMoBoz4w8/EdNYQsbGEjNxYSljXTEsA9O1pLTvAAAAAElF'
+                  'TkSuQmCC',
+    'text.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAD1BMVEX/'
+                '///M//+ZmZkzMzMAAABVsTOVAAAAAnRSTlP/AOW3MEoAAABISURBVHja'
+                'tcrRCgAgCENRbf7/N7dKomGvngjhMsPLD4NdMPwia438NRIyxsaL/XQZ'
+                'hyxpkC6zyjLXGVXnkhqWJWIIrOgeinECLlUCjBCqNQoAAAAASUVORK5C'
+                'YII=',
+    'unknown.png': 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAWCAMAAAD3n0w0AAAAD1BMVEX/'
+                   '///M//+ZmZkzMzMAAABVsTOVAAAAAnRSTlP/AOW3MEoAAABYSURBVHja'
+                   'ncvRDoAgDEPRruX/v1kmNHPBxMTLyzgD6FmsILg56g2hQnJkOco4yZhq'
+                   'tN5nYd5Zq0LsHblwxwP9GTCWsaGtoelANKzOlz/RfaLYUmLE6E28ALlN'
+                   'AupSdoFsAAAAAElFTkSuQmCC'}
+
+
+# Begin vendored code
+# This code is licensed under the Public Domain/CC0 and comes from
+# https://github.com/leenr/gzip-stream/blob/master/gzip_stream.py
+# Code was modified:
+#   removed type annotations to support python2.
+#   removed use of *, somearg for positional anonymous args.
+#   Default compression level to 9.
+
+class GZIPCompressedStream(io.RawIOBase):
+    def __init__(self, stream, compression_level=9):
+        assert 1 <= compression_level <= 9
+
+        self._compression_level = compression_level
+        self._stream = stream
+
+        self._compressed_stream = io.BytesIO()
+        self._compressor = gzip.GzipFile(
+            mode='wb',
+            fileobj=self._compressed_stream,
+            compresslevel=compression_level
+        )
+
+        # because of the GZIP header written by `GzipFile.__init__`:
+        self._compressed_stream.seek(0)
+
+    @property
+    def compression_level(self):
+        return self._compression_level
+
+    @property
+    def stream(self):
+        return self._stream
+
+    def readable(self):
+        return True
+
+    def _read_compressed_into(self, b):
+        buf = self._compressed_stream.read(len(b))
+        b[:len(buf)] = buf
+        return len(buf)
+
+    def readinto(self, b):
+        b = memoryview(b)
+
+        offset = 0
+        size = len(b)
+        while offset < size:
+            offset += self._read_compressed_into(b[offset:])
+            if offset < size:
+                # self._compressed_buffer now empty
+                if self._compressor.closed:
+                    # nothing to compress anymore
+                    break
+                # compress next bytes
+                self._read_n_compress(size)
+
+        return offset
+
+    def _read_n_compress(self, size):
+        assert size > 0
+
+        data = self._stream.read(size)
+
+        # rewind buffer to the start to free up memory
+        # (because anything currently in the buffer should be already
+        #  streamed off the object)
+        self._compressed_stream.seek(0)
+        self._compressed_stream.truncate(0)
+
+        if data:
+            self._compressor.write(data)
+        else:
+            # this will write final data (will flush zlib with Z_FINISH)
+            self._compressor.close()
+
+        # rewind to the buffer start
+        self._compressed_stream.seek(0)
+
+    def __repr__(self):
+        return (
+            '{self.__class__.__name__}('
+            '{self.stream!r}, '
+            'compression_level={self.compression_level!r}'
+            ')'
+        ).format(self=self)
+
+# End vendored code
+
+
+def get_mime_icon(mime, filename=''):
+    icon = (APACHE_FILE_ICON_MAP.get(filename) or
+            APACHE_MIME_ICON_MAP.get(mime) or
+            APACHE_MIME_ICON_MAP['_default'])
+    return "data:image/png;base64,%s" % ICON_IMAGES[icon]
+
+
+def get_cloud(cloud):
+    if isinstance(cloud, dict):
+        config = openstack.config.loader.OpenStackConfig().get_one(**cloud)
+        return openstack.connection.Connection(config=config)
+    else:
+        return openstack.connect(cloud=cloud)
+
+
+def retry_function(func):
+    for attempt in range(1, POST_ATTEMPTS + 1):
+        try:
+            return func()
+        except Exception:
+            if attempt >= POST_ATTEMPTS:
+                raise
+            else:
+                logging.exception("Error on attempt %d" % attempt)
+                time.sleep(attempt * 10)
+
+
+def sizeof_fmt(num, suffix='B'):
+    # From http://stackoverflow.com/questions/1094841/
+    # reusable-library-to-get-human-readable-version-of-file-size
+    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
+        if abs(num) < 1024.0:
+            return "%3.1f%s%s" % (num, unit, suffix)
+        num /= 1024.0
+    return "%.1f%s%s" % (num, 'Y', suffix)
+
+
+class FileDetail():
+    """
+    Used to generate indexes with links or as the file path
+    to push to swift.
+    """
+
+    def __init__(self, full_path, relative_path, filename=None):
+        """
+        Args:
+            full_path (str): The absolute path to the file on disk.
+            relative_path (str): The relative path from the artifacts source
+                                 used for links.
+            filename (str): An optional alternate filename in links.
+        """
+        # Make FileNotFoundError exception to be compatible with python2
+        try:
+            FileNotFoundError  # noqa: F823
+        except NameError:
+            FileNotFoundError = OSError
+
+        self.full_path = full_path
+        if filename is None:
+            self.filename = os.path.basename(full_path)
+        else:
+            self.filename = filename
+        self.relative_path = relative_path
+
+        if self.full_path and os.path.isfile(self.full_path):
+            mime_guess, encoding = mimetypes.guess_type(self.full_path)
+            self.mimetype = mime_guess if mime_guess else 'text/plain'
+            self.encoding = encoding
+            self.folder = False
+        else:
+            self.mimetype = 'application/directory'
+            self.encoding = None
+            self.folder = True
+        try:
+            st = os.stat(self.full_path)
+            self.last_modified = time.gmtime(st[stat.ST_MTIME])
+            self.size = st[stat.ST_SIZE]
+        except (FileNotFoundError, TypeError):
+            self.last_modified = time.gmtime(0)
+            self.size = 0
+
+    def __repr__(self):
+        t = 'Folder' if self.folder else 'File'
+        return '<%s %s>' % (t, self.relative_path)
+
+
+class FileList(Sequence):
+    '''A collection of FileDetail objects
+
+    This is a list-like group of FileDetail objects, intended to be
+    used as a context manager around the upload process.
+    '''
+    def __init__(self):
+        self.file_list = []
+        self.file_list.append(FileDetail(None, '', ''))
+        self.tempdirs = []
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        for tempdir in self.tempdirs:
+            shutil.rmtree(tempdir)
+
+    def __getitem__(self, item):
+        return self.file_list.__getitem__(item)
+
+    def __len__(self):
+        return self.file_list.__len__()
+
+    def get_tempdir(self):
+        '''Get a temporary directory
+
+        Returns path to a private temporary directory which will be
+        cleaned on exit
+        '''
+        tempdir = tempfile.mkdtemp(prefix='s-u-l-tmp')
+        self.tempdirs.append(tempdir)
+        return tempdir
+
+    @staticmethod
+    def _path_in_tree(root, path):
+        full_path = os.path.realpath(os.path.abspath(
+            os.path.expanduser(path)))
+        if not full_path.startswith(root):
+            logging.debug("Skipping path outside root: %s" % (path,))
+            return False
+        return True
+
+    def add(self, file_path):
+        """
+        Generate a list of files to upload to swift. Recurses through
+        directories
+        """
+
+        # file_list: A list of FileDetails to push to swift
+        file_list = []
+
+        if os.path.isfile(file_path):
+            relative_path = os.path.basename(file_path)
+            file_list.append(FileDetail(file_path, relative_path))
+        elif os.path.isdir(file_path):
+            original_root = os.path.realpath(os.path.abspath(
+                os.path.expanduser(file_path)))
+
+            parent_dir = os.path.dirname(file_path)
+            if not file_path.endswith('/'):
+                filename = os.path.basename(file_path)
+                full_path = file_path
+                relative_name = os.path.relpath(full_path, parent_dir)
+                file_list.append(FileDetail(full_path, relative_name,
+                                            filename))
+            # TODO: this will copy the result of symlinked files, but
+            # it won't follow directory symlinks.  If we add that, we
+            # should ensure that we don't loop.
+            for path, folders, files in os.walk(file_path):
+                # Sort folder in-place so that we recurse in order.
+                files.sort(key=lambda x: x.lower())
+                folders.sort(key=lambda x: x.lower())
+                # relative_path: The path between the given directory
+                # and the one being currently walked.
+                relative_path = os.path.relpath(path, parent_dir)
+
+                for filename in folders:
+                    full_path = os.path.join(path, filename)
+                    if not self._path_in_tree(original_root, full_path):
+                        continue
+                    relative_name = os.path.relpath(full_path, parent_dir)
+                    file_list.append(FileDetail(full_path, relative_name,
+                                                filename))
+
+                for filename in files:
+                    full_path = os.path.join(path, filename)
+                    if not self._path_in_tree(original_root, full_path):
+                        continue
+                    relative_name = os.path.relpath(full_path, parent_dir)
+                    file_detail = FileDetail(full_path, relative_name)
+                    file_list.append(file_detail)
+
+        self.file_list += file_list
+
+
+class Indexer():
+    """Index a FileList
+
+    Functions to generate indexes and other collated data for a
+    FileList
+
+     - make_indexes() : make index.html in folders
+    """
+    def __init__(self, file_list):
+        '''
+        Args:
+            file_list (FileList): A FileList object with all files
+             to be indexed.
+        '''
+        assert isinstance(file_list, FileList)
+        self.file_list = file_list
+
+    def _make_index_file(self, folder_links, title, tempdir, append_footer):
+        """Writes an index into a file for pushing"""
+        for file_details in folder_links:
+            # Do not generate an index file if one exists already.
+            # This may be the case when uploading other machine generated
+            # content like python coverage info.
+            if self.index_filename == file_details.filename:
+                return
+        index_content = self._generate_log_index(
+            folder_links, title, append_footer)
+        fd = open(os.path.join(tempdir, self.index_filename), 'w')
+        fd.write(index_content)
+        return os.path.join(tempdir, self.index_filename)
+
+    def _generate_log_index(self, folder_links, title, append_footer):
+        """Create an index of logfiles and links to them"""
+
+        output = '<html><head><title>%s</title></head><body>\n' % title
+        output += '<h1>%s</h1>\n' % title
+        output += '<table><tr><th></th><th>Name</th><th>Last Modified</th>'
+        output += '<th>Size</th></tr>'
+
+        file_details_to_append = None
+        for file_details in folder_links:
+            output += '<tr>'
+            output += (
+                '<td><img alt="[ ]" title="%(m)s" src="%(i)s"></img></td>' % ({
+                    'm': file_details.mimetype,
+                    'i': get_mime_icon(file_details.mimetype,
+                                       file_details.filename),
+                }))
+            filename = file_details.filename
+            if file_details.folder:
+                filename += '/'
+            output += '<td><a href="%s">%s</a></td>' % (
+                urlparse.quote(filename),
+                filename)
+            output += '<td>%s</td>' % time.asctime(
+                file_details.last_modified)
+            size = sizeof_fmt(file_details.size, suffix='')
+            output += '<td style="text-align: right">%s</td>' % size
+            output += '</tr>\n'
+
+            if (append_footer and
+                append_footer in file_details.filename):
+                file_details_to_append = file_details
+
+        output += '</table>'
+
+        if file_details_to_append:
+            output += '<br /><hr />'
+            try:
+                with open(file_details_to_append.full_path, 'r') as f:
+                    output += f.read()
+            except IOError:
+                logging.exception("Error opening file for appending")
+
+        output += '</body></html>\n'
+        return output
+
+    def make_indexes(self, create_parent_links=True,
+                     create_topdir_parent_link=False,
+                     append_footer='index_footer.html'):
+        '''Make index.html files
+
+        Iterate the file list and crete index.html files for folders
+
+        Args:
+          create_parent_links (bool): Create parent links
+          create_topdir_parent_link (bool): Create topdir parent link
+          append_footer (str): Filename of a footer to append to each
+             generated page
+
+        Return:
+            No value, the self.file_list will be updated
+        '''
+        self.index_filename = 'index.html'
+
+        folders = collections.OrderedDict()
+        for f in self.file_list:
+            if f.folder:
+                folders[f.relative_path] = []
+                folder = os.path.dirname(os.path.dirname(
+                    f.relative_path + '/'))
+                if folder == '/':
+                    folder = ''
+            else:
+                folder = os.path.dirname(f.relative_path)
+            folders[folder].append(f)
+
+        indexes = {}
+        parent_file_detail = FileDetail(None, '..', '..')
+        for folder, files in folders.items():
+            # Don't add the pseudo-top-directory
+            if files and files[0].full_path is None:
+                files = files[1:]
+                if create_topdir_parent_link:
+                    files = [parent_file_detail] + files
+            elif create_parent_links:
+                files = [parent_file_detail] + files
+
+            # Do generate a link to the parent directory
+            full_path = self._make_index_file(files, 'Index of %s' % (folder,),
+                                              self.file_list.get_tempdir(),
+                                              append_footer)
+
+            if full_path:
+                filename = os.path.basename(full_path)
+                relative_name = os.path.join(folder, filename)
+                indexes[folder] = FileDetail(full_path, relative_name)
+
+        # This appends the index file at the end of the group of files
+        # for each directory.
+        new_list = []
+        last_dirname = None
+        for f in reversed(list(self.file_list)):
+            if f.folder:
+                relative_path = f.relative_path + '/'
+            else:
+                relative_path = f.relative_path
+            dirname = os.path.dirname(relative_path)
+            if dirname == '/':
+                dirname = ''
+            if dirname != last_dirname:
+                index = indexes.pop(dirname, None)
+                if index:
+                    new_list.append(index)
+                    last_dirname = dirname
+            new_list.append(f)
+        new_list.reverse()
+        self.file_list.file_list = new_list
+
+
+class GzipFilter():
+    chunk_size = 16384
+
+    def __init__(self, infile):
+        self.gzipfile = GZIPCompressedStream(infile)
+        self.done = False
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        if self.done:
+            self.gzipfile.close()
+            raise StopIteration()
+        data = self.gzipfile.read(self.chunk_size)
+        if not data:
+            self.done = True
+        return data
+
+
+class DeflateFilter():
+    chunk_size = 16384
+
+    def __init__(self, infile):
+        self.infile = infile
+        self.encoder = zlib.compressobj()
+        self.done = False
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        if self.done:
+            raise StopIteration()
+        ret = b''
+        while True:
+            data = self.infile.read(self.chunk_size)
+            if data:
+                ret = self.encoder.compress(data)
+                if ret:
+                    break
+            else:
+                self.done = True
+                ret = self.encoder.flush()
+                break
+        return ret
+
+
+class Uploader():
+    def __init__(self, cloud, container, prefix=None, delete_after=None,
+                 public=True):
+        self.cloud = cloud
+        self.container = container
+        self.prefix = prefix or ''
+        self.delete_after = delete_after
+
+        sess = self.cloud.config.get_session()
+        adapter = requests.adapters.HTTPAdapter(pool_maxsize=100)
+        sess.mount('https://', adapter)
+
+        # If we're in Rackspace, there's some non-standard stuff we
+        # need to do to get the public endpoint.
+        try:
+            cdn_endpoint = self.cloud.session.auth.get_endpoint(
+                self.cloud.session, service_type='rax:object-cdn',
+                region_name=self.cloud.config.region_name,
+                interface=self.cloud.config.interface)
+            cdn_url = os.path.join(cdn_endpoint, self.container)
+        except keystoneauth1.exceptions.catalog.EndpointNotFound:
+            cdn_url = None
+
+        # We retry here because sometimes we get HTTP 401 errors in rax.
+        # They seem to happen infrequently (on the order of once a day across
+        # all jobs) so a retry is likely to work.
+        container = retry_function(
+            lambda: self.cloud.get_container(self.container))
+        if not container:
+            retry_function(
+                lambda: self.cloud.create_container(
+                    name=self.container, public=public))
+            headers = {'X-Container-Meta-Web-Index': 'index.html',
+                       'X-Container-Meta-Access-Control-Allow-Origin': '*'}
+            retry_function(
+                lambda: self.cloud.update_container(
+                    name=self.container,
+                    headers=headers))
+            # 'X-Container-Meta-Web-Listings': 'true'
+
+            # The ceph radosgw swift implementation requires an
+            # index.html at the root in order for any other indexes to
+            # work.
+            index_headers = {'access-control-allow-origin': '*'}
+            retry_function(
+                lambda: self.cloud.create_object(self.container,
+                                                 name='index.html',
+                                                 data='',
+                                                 content_type='text/html',
+                                                 **index_headers))
+
+            # Enable the CDN in rax
+            if cdn_url:
+                retry_function(lambda: self.cloud.session.put(cdn_url))
+
+        if cdn_url:
+            endpoint = retry_function(
+                lambda: self.cloud.session.head(
+                    cdn_url).headers['X-Cdn-Ssl-Uri'])
+            container = endpoint
+        else:
+            endpoint = self.cloud.object_store.get_endpoint()
+            container = os.path.join(endpoint, self.container)
+
+        self.url = os.path.join(container, self.prefix)
+
+    def upload(self, file_list):
+        """Spin up thread pool to upload to swift"""
+        num_threads = min(len(file_list), MAX_UPLOAD_THREADS)
+        threads = []
+        queue = queuelib.Queue()
+        # add items to queue
+        for f in file_list:
+            queue.put(f)
+
+        for x in range(num_threads):
+            t = threading.Thread(target=self.post_thread, args=(queue,))
+            threads.append(t)
+            t.start()
+
+        for t in threads:
+            t.join()
+
+    def post_thread(self, queue):
+        while True:
+            try:
+                file_detail = queue.get_nowait()
+                logging.debug("%s: processing job %s",
+                              threading.current_thread(),
+                              file_detail)
+                retry_function(lambda: self._post_file(file_detail))
+            except requests.exceptions.RequestException:
+                # Do our best to attempt to upload all the files
+                logging.exception("Error posting file after multiple attempts")
+                continue
+            except IOError:
+                # Do our best to attempt to upload all the files
+                logging.exception("Error opening file")
+                continue
+            except queuelib.Empty:
+                # No more work to do
+                return
+
+    @staticmethod
+    def _is_text_type(mimetype):
+        # We want to compress all text types.
+        if mimetype.startswith('text/'):
+            return True
+
+        # Further compress types that typically contain text but are no
+        # text sub type.
+        compress_types = [
+            'application/json',
+            'image/svg+xml',
+        ]
+        if mimetype in compress_types:
+            return True
+        return False
+
+    def _post_file(self, file_detail):
+        relative_path = os.path.join(self.prefix, file_detail.relative_path)
+        headers = {}
+        if self.delete_after:
+            headers['x-delete-after'] = str(self.delete_after)
+        headers['content-type'] = file_detail.mimetype
+        # This is required for Rackspace CDN
+        headers['access-control-allow-origin'] = '*'
+
+        if not file_detail.folder:
+            if (file_detail.encoding is None and
+                self._is_text_type(file_detail.mimetype)):
+                headers['content-encoding'] = 'gzip'
+                data = GzipFilter(open(file_detail.full_path, 'rb'))
+            else:
+                if file_detail.encoding:
+                    headers['content-encoding'] = file_detail.encoding
+                data = open(file_detail.full_path, 'rb')
+        else:
+            data = ''
+            relative_path = relative_path.rstrip('/')
+            if relative_path == '':
+                relative_path = '/'
+        self.cloud.create_object(self.container,
+                                 name=relative_path,
+                                 data=data,
+                                 **headers)
+
+
+def run(cloud, container, files,
+        indexes=True, parent_links=True, topdir_parent_link=False,
+        partition=False, footer='index_footer.html', delete_after=15552000,
+        prefix=None, public=True, dry_run=False):
+
+    if prefix:
+        prefix = prefix.lstrip('/')
+    if partition and prefix:
+        parts = prefix.split('/')
+        if len(parts) > 1:
+            container += '_' + parts[0]
+            prefix = '/'.join(parts[1:])
+
+    # Create the objects to make sure the arguments are sound.
+    with FileList() as file_list:
+        # Scan the files.
+        for file_path in files:
+            file_list.add(file_path)
+
+        indexer = Indexer(file_list)
+
+        # (Possibly) make indexes.
+        if indexes:
+            indexer.make_indexes(create_parent_links=parent_links,
+                                 create_topdir_parent_link=topdir_parent_link,
+                                 append_footer=footer)
+
+        logging.debug("List of files prepared to upload:")
+        for x in file_list:
+            logging.debug(x)
+
+        # Do no connect to swift or do any uploading in a dry run
+        if dry_run:
+            # No URL is known, so return nothing
+            return
+
+        # Upload.
+        uploader = Uploader(cloud, container, prefix, delete_after,
+                            public)
+        uploader.upload(file_list)
+        return uploader.url
+
+
+def ansible_main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            cloud=dict(required=True, type='raw'),
+            container=dict(required=True, type='str'),
+            files=dict(required=True, type='list'),
+            partition=dict(type='bool', default=False),
+            indexes=dict(type='bool', default=True),
+            parent_links=dict(type='bool', default=True),
+            topdir_parent_link=dict(type='bool', default=False),
+            public=dict(type='bool', default=True),
+            footer=dict(type='str'),
+            delete_after=dict(type='int'),
+            prefix=dict(type='str'),
+        )
+    )
+
+    p = module.params
+    cloud = get_cloud(p.get('cloud'))
+    try:
+        url = run(cloud, p.get('container'), p.get('files'),
+                  indexes=p.get('indexes'),
+                  parent_links=p.get('parent_links'),
+                  topdir_parent_link=p.get('topdir_parent_link'),
+                  partition=p.get('partition'),
+                  footer=p.get('footer'),
+                  delete_after=p.get('delete_after', 15552000),
+                  prefix=p.get('prefix'),
+                  public=p.get('public'))
+    except (keystoneauth1.exceptions.http.HttpError,
+            requests.exceptions.RequestException):
+        s = "Error uploading to %s.%s" % (cloud.name, cloud.config.region_name)
+        logging.exception(s)
+        s += "\n" + traceback.format_exc()
+        module.fail_json(
+            changed=False,
+            msg=s,
+            cloud=cloud.name,
+            region_name=cloud.config.region_name)
+    module.exit_json(changed=True,
+                     url=url)
+
+
+def cli_main():
+    parser = argparse.ArgumentParser(
+        description="Upload files to swift"
+    )
+    parser.add_argument('--verbose', action='store_true',
+                        help='show debug information')
+    parser.add_argument('--no-indexes', action='store_true',
+                        help='do not generate any indexes at all')
+    parser.add_argument('--no-parent-links', action='store_true',
+                        help='do not include links back to a parent dir')
+    parser.add_argument('--create-topdir-parent-link', action='store_true',
+                        help='include a link in the root directory of the '
+                             'files to the parent directory which may be the '
+                             'index of all results')
+    parser.add_argument('--no-public', action='store_true',
+                        help='do not create the container as public')
+    parser.add_argument('--partition', action='store_true',
+                        help='partition the prefix into multiple containers')
+    parser.add_argument('--append-footer', default='index_footer.html',
+                        help='when generating an index, if the given file is '
+                             'present in a directory, append it to the index '
+                             '(set to "none" to disable)')
+    parser.add_argument('--delete-after', default=15552000,
+                        help='Number of seconds to delete object after '
+                             'upload. Default is 6 months (15552000 seconds) '
+                             'and if set to 0 X-Delete-After will not be set',
+                        type=int)
+    parser.add_argument('--prefix',
+                        help='Prepend this path to the object names when '
+                             'uploading')
+    parser.add_argument('--dry-run', action='store_true',
+                        help='do not attempt to create containers or upload, '
+                             'useful with --verbose for debugging')
+    parser.add_argument('cloud',
+                        help='Name of the cloud to use when uploading')
+    parser.add_argument('container',
+                        help='Name of the container to use when uploading')
+    parser.add_argument('files', nargs='+',
+                        help='the file(s) to upload with recursive glob '
+                        'matching when supplied as a string')
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+        # Set requests log level accordingly
+        logging.getLogger("requests").setLevel(logging.DEBUG)
+        # logging.getLogger("keystoneauth").setLevel(logging.INFO)
+        # logging.getLogger("stevedore").setLevel(logging.INFO)
+        logging.captureWarnings(True)
+
+    append_footer = args.append_footer
+    if append_footer.lower() == 'none':
+        append_footer = None
+
+    url = run(get_cloud(args.cloud), args.container, args.files,
+              indexes=not args.no_indexes,
+              parent_links=not args.no_parent_links,
+              topdir_parent_link=args.create_topdir_parent_link,
+              partition=args.partition,
+              footer=append_footer,
+              delete_after=args.delete_after,
+              prefix=args.prefix,
+              public=not args.no_public,
+              dry_run=args.dry_run)
+    print(url)
+
+
+if __name__ == '__main__':
+    # Avoid unactionable warnings
+    requestsexceptions.squelch_warnings(
+        requestsexceptions.InsecureRequestWarning)
+
+    if sys.stdin.isatty():
+        cli_main()
+    else:
+        ansible_main()

roles/test-upload-logs-swift/tasks/main.yaml

@@ -0,0 +1,39 @@
+- name: Set zuul-log-path fact
+  include_role:
+    name: set-zuul-log-path-fact
+  when: zuul_log_path is not defined
+
+# Always upload (true), never upload (false) or only on failure ('failure')
+- when: zuul_site_upload_logs | default(true) | bool or
+        (zuul_site_upload_logs == 'failure' and not zuul_success | bool)
+  block:
+    # Use chmod instead of file because ansible 2.5 file with recurse and
+    # follow can't really handle symlinks to .
+    - name: Ensure logs are readable before uploading
+      delegate_to: localhost
+      command: "chmod -R u=rwX,g=rX,o=rX {{ zuul.executor.log_root }}/"
+      # ANSIBLE0007 chmod used in place of argument mode to file
+      tags:
+        - skip_ansible_lint
+
+    - name: Upload logs to swift
+      delegate_to: localhost
+      zuul_swift_upload:
+        cloud: "{{ zuul_log_cloud_config }}"
+        partition: "{{ zuul_log_partition }}"
+        container: "{{ zuul_log_container }}"
+        public: "{{ zuul_log_container_public }}"
+        prefix: "{{ zuul_log_path }}"
+        indexes: "{{ zuul_log_create_indexes }}"
+        files:
+          - "{{ zuul.executor.log_root }}/"
+        delete_after: "{{ zuul_log_delete_after | default(omit) }}"
+      register: upload_results
+
+- name: Return log URL to Zuul
+  delegate_to: localhost
+  zuul_return:
+    data:
+      zuul:
+        log_url: "{{ upload_results.url }}/"
+  when: upload_results is defined