doc-tools unit tests

Renamed sitemap file to avoid module name conflict
when importing at the sitemap unittest

Added py.test tox environment

Change-Id: I94480e374b29802414b62591a51c04ecd804905e
Closes-Bug: #1387716

.gitignore

@@ -14,6 +14,7 @@ sdist
 # Unit test / coverage reports
 .coverage
 .tox
+.testrepository
 
 # pbr generates these
 AUTHORS

.testr.conf

@@ -0,0 +1,7 @@
+[DEFAULT]
+test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
+             OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
+             OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \
+             ${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION
+test_id_option=--load-list $IDFILE
+test_list_option=--list

HACKING.rst

@@ -9,8 +9,7 @@ openstack-doc-tools style commandments
 Running tests
 -------------
 
-So far there are no tests included with the package but a test suite
-would be welcome!
+So far there are some tests included with the package.
 
 The openstack-indexpage tool is used while building the OpenStack
 documentation repositories, test building of these repositories with

bin/doc-tools-update-cli-reference

@@ -89,6 +89,7 @@ branch=cli-reference
 git branch --list $branch && git branch -D $branch
 git checkout -b $branch
 mv ../output/${project}.rst "doc/cli-reference/source"
+rm -rf ../output
 version=$($project --version 2>&1)
 version=${version##*\)}
 git commit -a -m "[cli-ref] Update python-${project}client to ${version##* }"

os_doc_tools/commands.py

@@ -741,9 +741,6 @@ def document_single_project(os_command, output_dir, continue_on_error):
                                           ["--os-image-api-version", "1"],
                                           "_v1", " (v1)")
 
-    if os_command == 'glance':
-        out_file.write(".. include:: glance_property_keys.rst\n")
-
     print("Finished.\n")
     out_file.close()
     return True

os_doc_tools/test/test_index.py

@@ -0,0 +1,37 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+from os_doc_tools import index
+import unittest
+
+
+class TestGenerateIndex(unittest.TestCase):
+    def test_dir_created(self):
+        path = 'path'
+        with mock.patch.object(index, 'open'):
+            with mock.patch.object(index.os, 'mkdir') as mock_mkdir:
+                index.generate_index_file(path)
+        self.assertTrue(mock_mkdir.called)
+
+    def test_dir_not_created_when_exists(self):
+        path = 'path'
+        with mock.patch.object(index, 'open'):
+            with mock.patch.object(index.os, 'mkdir') as mock_mkdir:
+                with mock.patch.object(index.os.path, 'isdir',
+                                       returned_value=True):
+                    index.generate_index_file(path)
+        self.assertFalse(mock_mkdir.called)
+
+
+if __name__ == '__main__':
+    unittest.main()

os_doc_tools/test/test_jsoncheck.py

@@ -0,0 +1,98 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+from os_doc_tools import jsoncheck
+import unittest
+
+
+class MockOpen(object):
+
+    def read(self):
+        return "raw"
+
+    def write(self):
+        return True
+
+
+class TestFileFunctions(unittest.TestCase):
+
+    def test_indent_note(self):
+        note = "Hello\nWorld"
+        with mock.patch.object(jsoncheck.textwrap, 'fill') as mock_fill:
+            mock_fill.return_value = "Hello World"
+            jsoncheck._indent_note(note)
+            mock_fill.assert_any_call('Hello', initial_indent='    ',
+                                      subsequent_indent='            ',
+                                      width=80)
+            mock_fill.assert_any_call('World', initial_indent='    ',
+                                      subsequent_indent='            ',
+                                      width=80)
+
+    def test_get_demjson_diagnostics(self):
+        raw = "raw"
+
+        with mock.patch.object(jsoncheck.demjson, 'decode', return_value=True):
+            errstr = jsoncheck._get_demjson_diagnostics(raw)
+            self.assertTrue(errstr is None)
+
+        with mock.patch.object(jsoncheck.demjson, 'decode') as mock_decode:
+            mock_decode.side_effect = jsoncheck.demjson.JSONError(raw)
+            errstr = jsoncheck._get_demjson_diagnostics(raw)
+            expected_error_str = "     Error: raw"
+            self.assertEqual(errstr, expected_error_str)
+
+    def test_parse_json(self):
+        raw = "raw"
+        with mock.patch.object(jsoncheck.json, 'loads',
+                               return_value="Success"):
+            parsed = jsoncheck._parse_json(raw)
+        self.assertEqual(parsed, "Success")
+
+        with mock.patch.object(jsoncheck.json, 'loads') as mock_loads:
+            mock_loads.side_effect = ValueError()
+            with self.assertRaises(jsoncheck.ParserException):
+                parsed = jsoncheck._parse_json(raw)
+
+    def test_format_parsed_json(self):
+        with mock.patch.object(jsoncheck.json, 'dumps') as mock_dumps:
+            mock_dumps.return_value = "Success"
+            returned_value = jsoncheck._format_parsed_json('raw')
+        self.assertEqual(returned_value, "Success\n")
+        self.assertTrue(mock_dumps.called)
+
+    def test_process_file(self):
+        with mock.patch.object(jsoncheck, 'open', returned_value=MockOpen()):
+            with mock.patch.object(jsoncheck, '_parse_json') as mock_parse:
+                mock_parse.side_effect = jsoncheck.ParserException
+                with self.assertRaises(ValueError):
+                    jsoncheck._process_file('path')
+
+        with mock.patch.object(jsoncheck, 'open', returned_value=MockOpen()):
+            with mock.patch.object(jsoncheck, '_parse_json',
+                                   returned_value="Success"):
+                with mock.patch.object(jsoncheck, '_format_parsed_json',
+                                       returned_value="not_raw"):
+                        with self.assertRaises(ValueError):
+                            jsoncheck._process_file('path', 'check')
+
+        with mock.patch.object(jsoncheck, 'open', returned_value=MockOpen()):
+            with mock.patch.object(jsoncheck, '_parse_json',
+                                   returned_value="Success"):
+                with mock.patch.object(jsoncheck, '_format_parsed_json',
+                                       returned_value="not_raw"):
+                    with self.assertRaises(ValueError):
+                        jsoncheck._process_file('path', 'formatting')
+
+
+if __name__ == '__main__':
+    unittest.main()

sitemap/generator/spiders/sitemap.py → sitemap/generator/spiders/sitemap_file.py

@@ -13,9 +13,9 @@
 import time
 import urlparse
 
-from generator import items
 from scrapy.linkextractors import LinkExtractor
 from scrapy import spiders
+from sitemap.generator import items
 
 
 class SitemapSpider(spiders.CrawlSpider):

sitemap/test/generator/spiders/test_sitemap_file.py

@@ -0,0 +1,110 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+from sitemap.generator.spiders import sitemap_file
+import unittest
+
+
+class TestSitemapSpider(unittest.TestCase):
+
+    def setUp(self):
+        self.spider = sitemap_file.SitemapSpider()
+
+    def test_set_vars_on_init(self):
+        domain = 'docs.openstack.org'
+        self.assertEqual(self.spider.domain, domain)
+        self.assertEqual(self.spider.allowed_domains, [domain])
+        self.assertEqual(self.spider.start_urls, ['http://%s' % domain])
+
+    def test_start_urls_get_appended(self):
+        urls = 'new.openstack.org, old.openstack.org'
+        urls_len = len(urls.split(','))
+        spider_len = len(self.spider.start_urls)
+
+        spider_with_urls = sitemap_file.SitemapSpider(urls=urls)
+        spider_with_urls_len = len(spider_with_urls.start_urls)
+
+        self.assertEqual(spider_with_urls_len, (urls_len + spider_len))
+
+    def test_parse_items_inits_sitemap(self):
+        response = mock.MagicMock()
+        with mock.patch.object(sitemap_file.items,
+                               'SitemapItem') as mocked_sitemap_item:
+            with mock.patch.object(sitemap_file, 'time'):
+                self.spider.parse_item(response)
+
+        self.assertTrue(mocked_sitemap_item.called)
+
+    def test_parse_items_gets_path(self):
+        response = mock.MagicMock()
+        with mock.patch.object(sitemap_file.items, 'SitemapItem'):
+            with mock.patch.object(sitemap_file.urlparse,
+                                   'urlsplit') as mocked_urlsplit:
+                with mock.patch.object(sitemap_file, 'time'):
+                    self.spider.parse_item(response)
+
+        self.assertTrue(mocked_urlsplit.called)
+
+    def test_parse_items_low_priority_weekly_freq(self):
+        response = mock.MagicMock()
+        path = sitemap_file.urlparse.SplitResult(
+            scheme='https',
+            netloc='docs.openstack.com',
+            path='/kilo',
+            query='',
+            fragment=''
+        )
+        with mock.patch.object(sitemap_file.urlparse, 'urlsplit',
+                               return_value=path):
+            with mock.patch.object(sitemap_file, 'time'):
+                returned_item = self.spider.parse_item(response)
+
+        self.assertEqual('0.5', returned_item['priority'])
+        self.assertEqual('weekly', returned_item['changefreq'])
+
+    def test_parse_items_high_priority_daily_freq(self):
+        response = mock.MagicMock()
+        path = sitemap_file.urlparse.SplitResult(
+            scheme='https',
+            netloc='docs.openstack.com',
+            path='/mitaka',
+            query='',
+            fragment=''
+        )
+        with mock.patch.object(sitemap_file.urlparse, 'urlsplit',
+                               return_value=path):
+            with mock.patch.object(sitemap_file, 'time'):
+                returned_item = self.spider.parse_item(response)
+
+        self.assertEqual('1.0', returned_item['priority'])
+        self.assertEqual('daily', returned_item['changefreq'])
+
+    def test_parse_returns_populated_item(self):
+        response = mock.MagicMock()
+        path = sitemap_file.urlparse.SplitResult(
+            scheme='https',
+            netloc='docs.openstack.com',
+            path='/mitaka',
+            query='',
+            fragment=''
+        )
+        with mock.patch.object(sitemap_file.urlparse, 'urlsplit',
+                               return_value=path):
+            with mock.patch.object(sitemap_file, 'time'):
+                returned_item = self.spider.parse_item(response)
+
+        self.assertEqual(4, len(returned_item))
+
+
+if __name__ == '__main__':
+    unittest.main()

sitemap/test/generator/test_items.py

@@ -0,0 +1,37 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+from sitemap.generator import items
+import unittest
+
+
+class TestSitemapItem(unittest.TestCase):
+
+    def test_class_type(self):
+        self.assertTrue(type(items.SitemapItem) is items.scrapy.item.ItemMeta)
+
+    def test_class_supports_fields(self):
+        with mock.patch.object(items.scrapy.item, 'Field'):
+            a = items.SitemapItem()
+
+        supported_fields = ['loc', 'lastmod', 'priority', 'changefreq']
+        for field in supported_fields:
+            a[field] = field
+
+        not_supported_fields = ['some', 'random', 'fields']
+        for field in not_supported_fields:
+            with self.assertRaises(KeyError):
+                a[field] = field
+
+if __name__ == '__main__':
+    unittest.main()

sitemap/test/generator/test_pipelines.py

@@ -0,0 +1,202 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+from sitemap.generator import pipelines
+import unittest
+
+
+class TestSitemapItemExporter(unittest.TestCase):
+
+    def test_start_exporting(self):
+        output = mock.MagicMock()
+        itemExplorer = pipelines.SitemapItemExporter(output)
+
+        with mock.patch.object(itemExplorer.xg, 'startDocument',
+                               return_value=None) as mock_start_document:
+            with mock.patch.object(itemExplorer.xg, 'startElement',
+                                   return_value=None) as mock_start_element:
+                itemExplorer.start_exporting()
+
+        self.assertTrue(mock_start_document.called)
+        self.assertTrue(mock_start_element.called)
+
+
+class TestIgnoreDuplicateUrls(unittest.TestCase):
+
+    def setUp(self):
+        self.ignore_urls = pipelines.IgnoreDuplicateUrls()
+
+    def test_set_is_set_at_init(self):
+        self.assertTrue(isinstance(self.ignore_urls.processed, set))
+
+    def test_set_is_empty_at_init(self):
+        self.assertEqual(len(self.ignore_urls.processed), 0)
+
+    def test_duplicate_url(self):
+        self.ignore_urls.processed.add('url')
+        item = {'loc': 'url'}
+        spider = mock.MagicMock()
+
+        with self.assertRaises(pipelines.scrapy.exceptions.DropItem):
+            self.ignore_urls.process_item(item, spider)
+
+    def test_url_added_to_processed(self):
+        self.assertFalse('url' in self.ignore_urls.processed)
+
+        item = {'loc': 'url'}
+        spider = mock.MagicMock()
+        self.ignore_urls.process_item(item, spider)
+        self.assertTrue('url' in self.ignore_urls.processed)
+
+    def test_item_is_returned(self):
+        item = {'loc': 'url'}
+        spider = mock.MagicMock()
+        returned_item = self.ignore_urls.process_item(item, spider)
+        self.assertEqual(item, returned_item)
+
+
+class TestExportSitemap(unittest.TestCase):
+
+    def setUp(self):
+        self.export_sitemap = pipelines.ExportSitemap()
+        self.spider = mock.MagicMock()
+
+    def test_variables_set_at_init(self):
+        self.assertTrue(isinstance(self.export_sitemap.files, dict))
+        self.assertTrue(self.export_sitemap.exporter is None)
+
+    def test_spider_opened_calls_open(self):
+        with mock.patch.object(pipelines, 'open',
+                               return_value=None) as mocked_open:
+            with mock.patch.object(pipelines,
+                                   'SitemapItemExporter'):
+                self.export_sitemap.spider_opened(self.spider)
+
+        self.assertTrue(mocked_open.called)
+
+    def test_spider_opened_assigns_spider(self):
+        prev_len = len(self.export_sitemap.files)
+        with mock.patch.object(pipelines, 'open',
+                               return_value=None):
+            with mock.patch.object(pipelines,
+                                   'SitemapItemExporter'):
+                self.export_sitemap.spider_opened(self.spider)
+
+        after_len = len(self.export_sitemap.files)
+        self.assertTrue(after_len - prev_len, 1)
+
+    def test_spider_opened_instantiates_exporter(self):
+        with mock.patch.object(pipelines, 'open',
+                               return_value=None):
+            with mock.patch.object(pipelines,
+                                   'SitemapItemExporter') as mocked_exporter:
+                self.export_sitemap.spider_opened(self.spider)
+
+        self.assertTrue(mocked_exporter.called)
+
+    def test_spider_opened_exporter_starts_exporting(self):
+        with mock.patch.object(pipelines, 'open',
+                               return_value=None):
+            with mock.patch.object(pipelines.SitemapItemExporter,
+                                   'start_exporting') as mocked_start:
+                self.export_sitemap.spider_opened(self.spider)
+
+        self.assertTrue(mocked_start.called)
+
+    def test_spider_closed_calls_finish(self):
+        self.export_sitemap.exporter = mock.MagicMock()
+        self.export_sitemap.exporter.finish_exporting = mock.MagicMock()
+        self.export_sitemap.files[self.spider] = mock.MagicMock()
+
+        with mock.patch.object(pipelines, 'lxml'):
+            with mock.patch.object(pipelines, 'open'):
+                self.export_sitemap.spider_closed(self.spider)
+
+        self.assertTrue(self.export_sitemap.exporter.finish_exporting.called)
+
+    def test_spider_closed_pops_spider(self):
+        self.export_sitemap.exporter = mock.MagicMock()
+        self.export_sitemap.files[self.spider] = mock.MagicMock()
+
+        self.assertTrue(self.spider in self.export_sitemap.files)
+
+        with mock.patch.object(pipelines, 'lxml'):
+            with mock.patch.object(pipelines, 'open'):
+                self.export_sitemap.spider_closed(self.spider)
+
+        self.assertFalse(self.spider in self.export_sitemap.files)
+
+    def test_spider_closed_parses_with_lxml(self):
+        self.export_sitemap.exporter = mock.MagicMock()
+        self.export_sitemap.exporter.finish_exporting = mock.MagicMock()
+        self.export_sitemap.files[self.spider] = mock.MagicMock()
+
+        with mock.patch.object(pipelines.lxml, 'etree'):
+            with mock.patch.object(pipelines.lxml.etree,
+                                   'parse') as mocked_lxml_parse:
+                with mock.patch.object(pipelines, 'open'):
+                    self.export_sitemap.spider_closed(self.spider)
+
+        self.assertTrue(mocked_lxml_parse.called)
+
+    def test_spider_closed_opens_xml_files(self):
+        self.export_sitemap.exporter = mock.MagicMock()
+        self.export_sitemap.exporter.finish_exporting = mock.MagicMock()
+        self.export_sitemap.files[self.spider] = mock.MagicMock()
+
+        with mock.patch.object(pipelines, 'lxml'):
+            with mock.patch.object(pipelines, 'open') as mocked_open:
+                self.export_sitemap.spider_closed(self.spider)
+
+        self.assertTrue(mocked_open.called)
+
+    def test_spider_closed_writes_tree(self):
+        self.export_sitemap.exporter = mock.MagicMock()
+        self.export_sitemap.exporter.finish_exporting = mock.MagicMock()
+        self.export_sitemap.files[self.spider] = mock.MagicMock()
+
+        with mock.patch.object(pipelines.lxml, 'etree'):
+            with mock.patch.object(pipelines.lxml.etree,
+                                   'tostring') as mocked_lxml_tostring:
+                with mock.patch.object(pipelines, 'open'):
+                    self.export_sitemap.spider_closed(self.spider)
+
+        self.assertTrue(mocked_lxml_tostring.called)
+
+    def test_process_item_exports_item(self):
+        item = spider = self.export_sitemap.exporter = mock.MagicMock()
+        self.export_sitemap.exporter.export_item = mock.MagicMock()
+        self.export_sitemap.process_item(item, spider)
+
+        self.assertTrue(self.export_sitemap.exporter.export_item.called)
+
+    def test_process_item_returns_item(self):
+        spider = self.export_sitemap.exporter = mock.MagicMock()
+        item = {'random': 'item'}
+        returned_item = self.export_sitemap.process_item(item, spider)
+
+        self.assertEqual(item, returned_item)
+
+    def test_from_crawler_exists(self):
+        attr_exists = hasattr(pipelines.ExportSitemap, 'from_crawler')
+        attr_callable = callable(getattr(pipelines.ExportSitemap,
+                                         'from_crawler'))
+        self.assertTrue(attr_exists and attr_callable)
+
+    def test_from_crawler_assigns_pipeline(self):
+        crawler = mock.MagicMock()
+        pipelines.ExportSitemap.from_crawler(crawler)
+        # still thinking how to go about here.
+
+if __name__ == '__main__':
+    unittest.main()

test-requirements.txt

@@ -11,4 +11,8 @@ pylint==1.4.5 # GPLv2
 
 reno>=1.8.0 # Apache2
 oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
-sphinx!=1.3b1,<1.3,>=1.2.1 # BSD
+
+testrepository>=0.0.18 # Apache-2.0/BSD
+
+# mock object framework
+mock>=2.0 # BSD

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 minversion = 1.6
-envlist = py34,py27,pep8
+envlist = py27,pep8
 skipsdist = True
 
 [testenv]
@@ -9,6 +9,8 @@ install_command = pip install -U {opts} {packages}
 setenv =
    VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/test-requirements.txt
+       -r{toxinidir}/requirements.txt
+commands = python setup.py testr --slowest --testr-args='{posargs}'
 
 [testenv:pep8]
 commands =