From c050836a8f6a688a9477d3da1dd775e2fd376428 Mon Sep 17 00:00:00 2001 From: Percila Date: Wed, 24 Feb 2016 16:17:45 +0300 Subject: [PATCH] doc-tools unit tests Renamed sitemap file to avoid module name conflict when importing at the sitemap unittest Added py.test tox environment Change-Id: I94480e374b29802414b62591a51c04ecd804905e Closes-Bug: #1387716 --- .gitignore | 1 + .testr.conf | 7 + HACKING.rst | 3 +- bin/doc-tools-update-cli-reference | 1 + os_doc_tools/commands.py | 3 - os_doc_tools/test/__init__.py | 0 os_doc_tools/test/test_index.py | 37 ++++ os_doc_tools/test/test_jsoncheck.py | 98 +++++++++ sitemap/__init__.py | 0 .../spiders/{sitemap.py => sitemap_file.py} | 2 +- sitemap/test/__init__.py | 0 .../generator/spiders/test_sitemap_file.py | 110 ++++++++++ sitemap/test/generator/test_items.py | 37 ++++ sitemap/test/generator/test_pipelines.py | 202 ++++++++++++++++++ test-requirements.txt | 6 +- tox.ini | 4 +- 16 files changed, 503 insertions(+), 8 deletions(-) create mode 100644 .testr.conf create mode 100644 os_doc_tools/test/__init__.py create mode 100644 os_doc_tools/test/test_index.py create mode 100644 os_doc_tools/test/test_jsoncheck.py create mode 100644 sitemap/__init__.py rename sitemap/generator/spiders/{sitemap.py => sitemap_file.py} (98%) create mode 100644 sitemap/test/__init__.py create mode 100644 sitemap/test/generator/spiders/test_sitemap_file.py create mode 100644 sitemap/test/generator/test_items.py create mode 100644 sitemap/test/generator/test_pipelines.py diff --git a/.gitignore b/.gitignore index 0b420811..fddd160c 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ sdist # Unit test / coverage reports .coverage .tox +.testrepository # pbr generates these AUTHORS diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 00000000..6d83b3c4 --- /dev/null +++ b/.testr.conf @@ -0,0 +1,7 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ + OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ + OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ + ${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION +test_id_option=--load-list $IDFILE +test_list_option=--list diff --git a/HACKING.rst b/HACKING.rst index 877ddcbf..0c1e16e4 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -9,8 +9,7 @@ openstack-doc-tools style commandments Running tests ------------- -So far there are no tests included with the package but a test suite -would be welcome! +So far there are some tests included with the package. The openstack-indexpage tool is used while building the OpenStack documentation repositories, test building of these repositories with diff --git a/bin/doc-tools-update-cli-reference b/bin/doc-tools-update-cli-reference index a8707093..192b4841 100755 --- a/bin/doc-tools-update-cli-reference +++ b/bin/doc-tools-update-cli-reference @@ -89,6 +89,7 @@ branch=cli-reference git branch --list $branch && git branch -D $branch git checkout -b $branch mv ../output/${project}.rst "doc/cli-reference/source" +rm -rf ../output version=$($project --version 2>&1) version=${version##*\)} git commit -a -m "[cli-ref] Update python-${project}client to ${version##* }" diff --git a/os_doc_tools/commands.py b/os_doc_tools/commands.py index 341a3965..97c37364 100644 --- a/os_doc_tools/commands.py +++ b/os_doc_tools/commands.py @@ -741,9 +741,6 @@ def document_single_project(os_command, output_dir, continue_on_error): ["--os-image-api-version", "1"], "_v1", " (v1)") - if os_command == 'glance': - out_file.write(".. include:: glance_property_keys.rst\n") - print("Finished.\n") out_file.close() return True diff --git a/os_doc_tools/test/__init__.py b/os_doc_tools/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/os_doc_tools/test/test_index.py b/os_doc_tools/test/test_index.py new file mode 100644 index 00000000..e5adade7 --- /dev/null +++ b/os_doc_tools/test/test_index.py @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +from os_doc_tools import index +import unittest + + +class TestGenerateIndex(unittest.TestCase): + def test_dir_created(self): + path = 'path' + with mock.patch.object(index, 'open'): + with mock.patch.object(index.os, 'mkdir') as mock_mkdir: + index.generate_index_file(path) + self.assertTrue(mock_mkdir.called) + + def test_dir_not_created_when_exists(self): + path = 'path' + with mock.patch.object(index, 'open'): + with mock.patch.object(index.os, 'mkdir') as mock_mkdir: + with mock.patch.object(index.os.path, 'isdir', + returned_value=True): + index.generate_index_file(path) + self.assertFalse(mock_mkdir.called) + + +if __name__ == '__main__': + unittest.main() diff --git a/os_doc_tools/test/test_jsoncheck.py b/os_doc_tools/test/test_jsoncheck.py new file mode 100644 index 00000000..09bbd823 --- /dev/null +++ b/os_doc_tools/test/test_jsoncheck.py @@ -0,0 +1,98 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +from os_doc_tools import jsoncheck +import unittest + + +class MockOpen(object): + + def read(self): + return "raw" + + def write(self): + return True + + +class TestFileFunctions(unittest.TestCase): + + def test_indent_note(self): + note = "Hello\nWorld" + with mock.patch.object(jsoncheck.textwrap, 'fill') as mock_fill: + mock_fill.return_value = "Hello World" + jsoncheck._indent_note(note) + mock_fill.assert_any_call('Hello', initial_indent=' ', + subsequent_indent=' ', + width=80) + mock_fill.assert_any_call('World', initial_indent=' ', + subsequent_indent=' ', + width=80) + + def test_get_demjson_diagnostics(self): + raw = "raw" + + with mock.patch.object(jsoncheck.demjson, 'decode', return_value=True): + errstr = jsoncheck._get_demjson_diagnostics(raw) + self.assertTrue(errstr is None) + + with mock.patch.object(jsoncheck.demjson, 'decode') as mock_decode: + mock_decode.side_effect = jsoncheck.demjson.JSONError(raw) + errstr = jsoncheck._get_demjson_diagnostics(raw) + expected_error_str = " Error: raw" + self.assertEqual(errstr, expected_error_str) + + def test_parse_json(self): + raw = "raw" + with mock.patch.object(jsoncheck.json, 'loads', + return_value="Success"): + parsed = jsoncheck._parse_json(raw) + self.assertEqual(parsed, "Success") + + with mock.patch.object(jsoncheck.json, 'loads') as mock_loads: + mock_loads.side_effect = ValueError() + with self.assertRaises(jsoncheck.ParserException): + parsed = jsoncheck._parse_json(raw) + + def test_format_parsed_json(self): + with mock.patch.object(jsoncheck.json, 'dumps') as mock_dumps: + mock_dumps.return_value = "Success" + returned_value = jsoncheck._format_parsed_json('raw') + self.assertEqual(returned_value, "Success\n") + self.assertTrue(mock_dumps.called) + + def test_process_file(self): + with mock.patch.object(jsoncheck, 'open', returned_value=MockOpen()): + with mock.patch.object(jsoncheck, '_parse_json') as mock_parse: + mock_parse.side_effect = jsoncheck.ParserException + with self.assertRaises(ValueError): + jsoncheck._process_file('path') + + with mock.patch.object(jsoncheck, 'open', returned_value=MockOpen()): + with mock.patch.object(jsoncheck, '_parse_json', + returned_value="Success"): + with mock.patch.object(jsoncheck, '_format_parsed_json', + returned_value="not_raw"): + with self.assertRaises(ValueError): + jsoncheck._process_file('path', 'check') + + with mock.patch.object(jsoncheck, 'open', returned_value=MockOpen()): + with mock.patch.object(jsoncheck, '_parse_json', + returned_value="Success"): + with mock.patch.object(jsoncheck, '_format_parsed_json', + returned_value="not_raw"): + with self.assertRaises(ValueError): + jsoncheck._process_file('path', 'formatting') + + +if __name__ == '__main__': + unittest.main() diff --git a/sitemap/__init__.py b/sitemap/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sitemap/generator/spiders/sitemap.py b/sitemap/generator/spiders/sitemap_file.py similarity index 98% rename from sitemap/generator/spiders/sitemap.py rename to sitemap/generator/spiders/sitemap_file.py index 582ba66b..878823c6 100644 --- a/sitemap/generator/spiders/sitemap.py +++ b/sitemap/generator/spiders/sitemap_file.py @@ -13,9 +13,9 @@ import time import urlparse -from generator import items from scrapy.linkextractors import LinkExtractor from scrapy import spiders +from sitemap.generator import items class SitemapSpider(spiders.CrawlSpider): diff --git a/sitemap/test/__init__.py b/sitemap/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sitemap/test/generator/spiders/test_sitemap_file.py b/sitemap/test/generator/spiders/test_sitemap_file.py new file mode 100644 index 00000000..ea929452 --- /dev/null +++ b/sitemap/test/generator/spiders/test_sitemap_file.py @@ -0,0 +1,110 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +from sitemap.generator.spiders import sitemap_file +import unittest + + +class TestSitemapSpider(unittest.TestCase): + + def setUp(self): + self.spider = sitemap_file.SitemapSpider() + + def test_set_vars_on_init(self): + domain = 'docs.openstack.org' + self.assertEqual(self.spider.domain, domain) + self.assertEqual(self.spider.allowed_domains, [domain]) + self.assertEqual(self.spider.start_urls, ['http://%s' % domain]) + + def test_start_urls_get_appended(self): + urls = 'new.openstack.org, old.openstack.org' + urls_len = len(urls.split(',')) + spider_len = len(self.spider.start_urls) + + spider_with_urls = sitemap_file.SitemapSpider(urls=urls) + spider_with_urls_len = len(spider_with_urls.start_urls) + + self.assertEqual(spider_with_urls_len, (urls_len + spider_len)) + + def test_parse_items_inits_sitemap(self): + response = mock.MagicMock() + with mock.patch.object(sitemap_file.items, + 'SitemapItem') as mocked_sitemap_item: + with mock.patch.object(sitemap_file, 'time'): + self.spider.parse_item(response) + + self.assertTrue(mocked_sitemap_item.called) + + def test_parse_items_gets_path(self): + response = mock.MagicMock() + with mock.patch.object(sitemap_file.items, 'SitemapItem'): + with mock.patch.object(sitemap_file.urlparse, + 'urlsplit') as mocked_urlsplit: + with mock.patch.object(sitemap_file, 'time'): + self.spider.parse_item(response) + + self.assertTrue(mocked_urlsplit.called) + + def test_parse_items_low_priority_weekly_freq(self): + response = mock.MagicMock() + path = sitemap_file.urlparse.SplitResult( + scheme='https', + netloc='docs.openstack.com', + path='/kilo', + query='', + fragment='' + ) + with mock.patch.object(sitemap_file.urlparse, 'urlsplit', + return_value=path): + with mock.patch.object(sitemap_file, 'time'): + returned_item = self.spider.parse_item(response) + + self.assertEqual('0.5', returned_item['priority']) + self.assertEqual('weekly', returned_item['changefreq']) + + def test_parse_items_high_priority_daily_freq(self): + response = mock.MagicMock() + path = sitemap_file.urlparse.SplitResult( + scheme='https', + netloc='docs.openstack.com', + path='/mitaka', + query='', + fragment='' + ) + with mock.patch.object(sitemap_file.urlparse, 'urlsplit', + return_value=path): + with mock.patch.object(sitemap_file, 'time'): + returned_item = self.spider.parse_item(response) + + self.assertEqual('1.0', returned_item['priority']) + self.assertEqual('daily', returned_item['changefreq']) + + def test_parse_returns_populated_item(self): + response = mock.MagicMock() + path = sitemap_file.urlparse.SplitResult( + scheme='https', + netloc='docs.openstack.com', + path='/mitaka', + query='', + fragment='' + ) + with mock.patch.object(sitemap_file.urlparse, 'urlsplit', + return_value=path): + with mock.patch.object(sitemap_file, 'time'): + returned_item = self.spider.parse_item(response) + + self.assertEqual(4, len(returned_item)) + + +if __name__ == '__main__': + unittest.main() diff --git a/sitemap/test/generator/test_items.py b/sitemap/test/generator/test_items.py new file mode 100644 index 00000000..aab71f11 --- /dev/null +++ b/sitemap/test/generator/test_items.py @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +from sitemap.generator import items +import unittest + + +class TestSitemapItem(unittest.TestCase): + + def test_class_type(self): + self.assertTrue(type(items.SitemapItem) is items.scrapy.item.ItemMeta) + + def test_class_supports_fields(self): + with mock.patch.object(items.scrapy.item, 'Field'): + a = items.SitemapItem() + + supported_fields = ['loc', 'lastmod', 'priority', 'changefreq'] + for field in supported_fields: + a[field] = field + + not_supported_fields = ['some', 'random', 'fields'] + for field in not_supported_fields: + with self.assertRaises(KeyError): + a[field] = field + +if __name__ == '__main__': + unittest.main() diff --git a/sitemap/test/generator/test_pipelines.py b/sitemap/test/generator/test_pipelines.py new file mode 100644 index 00000000..1eec6b5d --- /dev/null +++ b/sitemap/test/generator/test_pipelines.py @@ -0,0 +1,202 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +from sitemap.generator import pipelines +import unittest + + +class TestSitemapItemExporter(unittest.TestCase): + + def test_start_exporting(self): + output = mock.MagicMock() + itemExplorer = pipelines.SitemapItemExporter(output) + + with mock.patch.object(itemExplorer.xg, 'startDocument', + return_value=None) as mock_start_document: + with mock.patch.object(itemExplorer.xg, 'startElement', + return_value=None) as mock_start_element: + itemExplorer.start_exporting() + + self.assertTrue(mock_start_document.called) + self.assertTrue(mock_start_element.called) + + +class TestIgnoreDuplicateUrls(unittest.TestCase): + + def setUp(self): + self.ignore_urls = pipelines.IgnoreDuplicateUrls() + + def test_set_is_set_at_init(self): + self.assertTrue(isinstance(self.ignore_urls.processed, set)) + + def test_set_is_empty_at_init(self): + self.assertEqual(len(self.ignore_urls.processed), 0) + + def test_duplicate_url(self): + self.ignore_urls.processed.add('url') + item = {'loc': 'url'} + spider = mock.MagicMock() + + with self.assertRaises(pipelines.scrapy.exceptions.DropItem): + self.ignore_urls.process_item(item, spider) + + def test_url_added_to_processed(self): + self.assertFalse('url' in self.ignore_urls.processed) + + item = {'loc': 'url'} + spider = mock.MagicMock() + self.ignore_urls.process_item(item, spider) + self.assertTrue('url' in self.ignore_urls.processed) + + def test_item_is_returned(self): + item = {'loc': 'url'} + spider = mock.MagicMock() + returned_item = self.ignore_urls.process_item(item, spider) + self.assertEqual(item, returned_item) + + +class TestExportSitemap(unittest.TestCase): + + def setUp(self): + self.export_sitemap = pipelines.ExportSitemap() + self.spider = mock.MagicMock() + + def test_variables_set_at_init(self): + self.assertTrue(isinstance(self.export_sitemap.files, dict)) + self.assertTrue(self.export_sitemap.exporter is None) + + def test_spider_opened_calls_open(self): + with mock.patch.object(pipelines, 'open', + return_value=None) as mocked_open: + with mock.patch.object(pipelines, + 'SitemapItemExporter'): + self.export_sitemap.spider_opened(self.spider) + + self.assertTrue(mocked_open.called) + + def test_spider_opened_assigns_spider(self): + prev_len = len(self.export_sitemap.files) + with mock.patch.object(pipelines, 'open', + return_value=None): + with mock.patch.object(pipelines, + 'SitemapItemExporter'): + self.export_sitemap.spider_opened(self.spider) + + after_len = len(self.export_sitemap.files) + self.assertTrue(after_len - prev_len, 1) + + def test_spider_opened_instantiates_exporter(self): + with mock.patch.object(pipelines, 'open', + return_value=None): + with mock.patch.object(pipelines, + 'SitemapItemExporter') as mocked_exporter: + self.export_sitemap.spider_opened(self.spider) + + self.assertTrue(mocked_exporter.called) + + def test_spider_opened_exporter_starts_exporting(self): + with mock.patch.object(pipelines, 'open', + return_value=None): + with mock.patch.object(pipelines.SitemapItemExporter, + 'start_exporting') as mocked_start: + self.export_sitemap.spider_opened(self.spider) + + self.assertTrue(mocked_start.called) + + def test_spider_closed_calls_finish(self): + self.export_sitemap.exporter = mock.MagicMock() + self.export_sitemap.exporter.finish_exporting = mock.MagicMock() + self.export_sitemap.files[self.spider] = mock.MagicMock() + + with mock.patch.object(pipelines, 'lxml'): + with mock.patch.object(pipelines, 'open'): + self.export_sitemap.spider_closed(self.spider) + + self.assertTrue(self.export_sitemap.exporter.finish_exporting.called) + + def test_spider_closed_pops_spider(self): + self.export_sitemap.exporter = mock.MagicMock() + self.export_sitemap.files[self.spider] = mock.MagicMock() + + self.assertTrue(self.spider in self.export_sitemap.files) + + with mock.patch.object(pipelines, 'lxml'): + with mock.patch.object(pipelines, 'open'): + self.export_sitemap.spider_closed(self.spider) + + self.assertFalse(self.spider in self.export_sitemap.files) + + def test_spider_closed_parses_with_lxml(self): + self.export_sitemap.exporter = mock.MagicMock() + self.export_sitemap.exporter.finish_exporting = mock.MagicMock() + self.export_sitemap.files[self.spider] = mock.MagicMock() + + with mock.patch.object(pipelines.lxml, 'etree'): + with mock.patch.object(pipelines.lxml.etree, + 'parse') as mocked_lxml_parse: + with mock.patch.object(pipelines, 'open'): + self.export_sitemap.spider_closed(self.spider) + + self.assertTrue(mocked_lxml_parse.called) + + def test_spider_closed_opens_xml_files(self): + self.export_sitemap.exporter = mock.MagicMock() + self.export_sitemap.exporter.finish_exporting = mock.MagicMock() + self.export_sitemap.files[self.spider] = mock.MagicMock() + + with mock.patch.object(pipelines, 'lxml'): + with mock.patch.object(pipelines, 'open') as mocked_open: + self.export_sitemap.spider_closed(self.spider) + + self.assertTrue(mocked_open.called) + + def test_spider_closed_writes_tree(self): + self.export_sitemap.exporter = mock.MagicMock() + self.export_sitemap.exporter.finish_exporting = mock.MagicMock() + self.export_sitemap.files[self.spider] = mock.MagicMock() + + with mock.patch.object(pipelines.lxml, 'etree'): + with mock.patch.object(pipelines.lxml.etree, + 'tostring') as mocked_lxml_tostring: + with mock.patch.object(pipelines, 'open'): + self.export_sitemap.spider_closed(self.spider) + + self.assertTrue(mocked_lxml_tostring.called) + + def test_process_item_exports_item(self): + item = spider = self.export_sitemap.exporter = mock.MagicMock() + self.export_sitemap.exporter.export_item = mock.MagicMock() + self.export_sitemap.process_item(item, spider) + + self.assertTrue(self.export_sitemap.exporter.export_item.called) + + def test_process_item_returns_item(self): + spider = self.export_sitemap.exporter = mock.MagicMock() + item = {'random': 'item'} + returned_item = self.export_sitemap.process_item(item, spider) + + self.assertEqual(item, returned_item) + + def test_from_crawler_exists(self): + attr_exists = hasattr(pipelines.ExportSitemap, 'from_crawler') + attr_callable = callable(getattr(pipelines.ExportSitemap, + 'from_crawler')) + self.assertTrue(attr_exists and attr_callable) + + def test_from_crawler_assigns_pipeline(self): + crawler = mock.MagicMock() + pipelines.ExportSitemap.from_crawler(crawler) + # still thinking how to go about here. + +if __name__ == '__main__': + unittest.main() diff --git a/test-requirements.txt b/test-requirements.txt index 0dddb5f0..8a0efbcc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,4 +11,8 @@ pylint==1.4.5 # GPLv2 reno>=1.8.0 # Apache2 oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0 -sphinx!=1.3b1,<1.3,>=1.2.1 # BSD + +testrepository>=0.0.18 # Apache-2.0/BSD + +# mock object framework +mock>=2.0 # BSD diff --git a/tox.ini b/tox.ini index 76f2095f..11a0ef78 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 1.6 -envlist = py34,py27,pep8 +envlist = py27,pep8 skipsdist = True [testenv] @@ -9,6 +9,8 @@ install_command = pip install -U {opts} {packages} setenv = VIRTUAL_ENV={envdir} deps = -r{toxinidir}/test-requirements.txt + -r{toxinidir}/requirements.txt +commands = python setup.py testr --slowest --testr-args='{posargs}' [testenv:pep8] commands =