Renamed sitemap file to avoid module name conflict when importing at the sitemap unittest Added py.test tox environment Change-Id: I94480e374b29802414b62591a51c04ecd804905e Closes-Bug: #1387716changes/22/284122/34
@ -0,0 +1,7 @@ | |||
[DEFAULT] | |||
test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ | |||
OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ | |||
OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ | |||
${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION | |||
test_id_option=--load-list $IDFILE | |||
test_list_option=--list |
@ -0,0 +1,37 @@ | |||
# Licensed under the Apache License, Version 2.0 (the "License"); you may | |||
# not use this file except in compliance with the License. You may obtain | |||
# a copy of the License at | |||
# | |||
# http://www.apache.org/licenses/LICENSE-2.0 | |||
# | |||
# Unless required by applicable law or agreed to in writing, software | |||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT | |||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | |||
# License for the specific language governing permissions and limitations | |||
# under the License. | |||
import mock | |||
from os_doc_tools import index | |||
import unittest | |||
class TestGenerateIndex(unittest.TestCase): | |||
def test_dir_created(self): | |||
path = 'path' | |||
with mock.patch.object(index, 'open'): | |||
with mock.patch.object(index.os, 'mkdir') as mock_mkdir: | |||
index.generate_index_file(path) | |||
self.assertTrue(mock_mkdir.called) | |||
def test_dir_not_created_when_exists(self): | |||
path = 'path' | |||
with mock.patch.object(index, 'open'): | |||
with mock.patch.object(index.os, 'mkdir') as mock_mkdir: | |||
with mock.patch.object(index.os.path, 'isdir', | |||
returned_value=True): | |||
index.generate_index_file(path) | |||
self.assertFalse(mock_mkdir.called) | |||
if __name__ == '__main__': | |||
unittest.main() |
@ -0,0 +1,98 @@ | |||
# Licensed under the Apache License, Version 2.0 (the "License"); you may | |||
# not use this file except in compliance with the License. You may obtain | |||
# a copy of the License at | |||
# | |||
# http://www.apache.org/licenses/LICENSE-2.0 | |||
# | |||
# Unless required by applicable law or agreed to in writing, software | |||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT | |||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | |||
# License for the specific language governing permissions and limitations | |||
# under the License. | |||
import mock | |||
from os_doc_tools import jsoncheck | |||
import unittest | |||
class MockOpen(object): | |||
def read(self): | |||
return "raw" | |||
def write(self): | |||
return True | |||
class TestFileFunctions(unittest.TestCase): | |||
def test_indent_note(self): | |||
note = "Hello\nWorld" | |||
with mock.patch.object(jsoncheck.textwrap, 'fill') as mock_fill: | |||
mock_fill.return_value = "Hello World" | |||
jsoncheck._indent_note(note) | |||
mock_fill.assert_any_call('Hello', initial_indent=' ', | |||
subsequent_indent=' ', | |||
width=80) | |||
mock_fill.assert_any_call('World', initial_indent=' ', | |||
subsequent_indent=' ', | |||
width=80) | |||
def test_get_demjson_diagnostics(self): | |||
raw = "raw" | |||
with mock.patch.object(jsoncheck.demjson, 'decode', return_value=True): | |||
errstr = jsoncheck._get_demjson_diagnostics(raw) | |||
self.assertTrue(errstr is None) | |||
with mock.patch.object(jsoncheck.demjson, 'decode') as mock_decode: | |||
mock_decode.side_effect = jsoncheck.demjson.JSONError(raw) | |||
errstr = jsoncheck._get_demjson_diagnostics(raw) | |||
expected_error_str = " Error: raw" | |||
self.assertEqual(errstr, expected_error_str) | |||
def test_parse_json(self): | |||
raw = "raw" | |||
with mock.patch.object(jsoncheck.json, 'loads', | |||
return_value="Success"): | |||
parsed = jsoncheck._parse_json(raw) | |||
self.assertEqual(parsed, "Success") | |||
with mock.patch.object(jsoncheck.json, 'loads') as mock_loads: | |||
mock_loads.side_effect = ValueError() | |||
with self.assertRaises(jsoncheck.ParserException): | |||
parsed = jsoncheck._parse_json(raw) | |||
def test_format_parsed_json(self): | |||
with mock.patch.object(jsoncheck.json, 'dumps') as mock_dumps: | |||
mock_dumps.return_value = "Success" | |||
returned_value = jsoncheck._format_parsed_json('raw') | |||
self.assertEqual(returned_value, "Success\n") | |||
self.assertTrue(mock_dumps.called) | |||
def test_process_file(self): | |||
with mock.patch.object(jsoncheck, 'open', returned_value=MockOpen()): | |||
with mock.patch.object(jsoncheck, '_parse_json') as mock_parse: | |||
mock_parse.side_effect = jsoncheck.ParserException | |||
with self.assertRaises(ValueError): | |||
jsoncheck._process_file('path') | |||
with mock.patch.object(jsoncheck, 'open', returned_value=MockOpen()): | |||
with mock.patch.object(jsoncheck, '_parse_json', | |||
returned_value="Success"): | |||
with mock.patch.object(jsoncheck, '_format_parsed_json', | |||
returned_value="not_raw"): | |||
with self.assertRaises(ValueError): | |||
jsoncheck._process_file('path', 'check') | |||
with mock.patch.object(jsoncheck, 'open', returned_value=MockOpen()): | |||
with mock.patch.object(jsoncheck, '_parse_json', | |||
returned_value="Success"): | |||
with mock.patch.object(jsoncheck, '_format_parsed_json', | |||
returned_value="not_raw"): | |||
with self.assertRaises(ValueError): | |||
jsoncheck._process_file('path', 'formatting') | |||
if __name__ == '__main__': | |||
unittest.main() |
@ -0,0 +1,110 @@ | |||
# Licensed under the Apache License, Version 2.0 (the "License"); you may | |||
# not use this file except in compliance with the License. You may obtain | |||
# a copy of the License at | |||
# | |||
# http://www.apache.org/licenses/LICENSE-2.0 | |||
# | |||
# Unless required by applicable law or agreed to in writing, software | |||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT | |||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | |||
# License for the specific language governing permissions and limitations | |||
# under the License. | |||
import mock | |||
from sitemap.generator.spiders import sitemap_file | |||
import unittest | |||
class TestSitemapSpider(unittest.TestCase): | |||
def setUp(self): | |||
self.spider = sitemap_file.SitemapSpider() | |||
def test_set_vars_on_init(self): | |||
domain = 'docs.openstack.org' | |||
self.assertEqual(self.spider.domain, domain) | |||
self.assertEqual(self.spider.allowed_domains, [domain]) | |||
self.assertEqual(self.spider.start_urls, ['http://%s' % domain]) | |||
def test_start_urls_get_appended(self): | |||
urls = 'new.openstack.org, old.openstack.org' | |||
urls_len = len(urls.split(',')) | |||
spider_len = len(self.spider.start_urls) | |||
spider_with_urls = sitemap_file.SitemapSpider(urls=urls) | |||
spider_with_urls_len = len(spider_with_urls.start_urls) | |||
self.assertEqual(spider_with_urls_len, (urls_len + spider_len)) | |||
def test_parse_items_inits_sitemap(self): | |||
response = mock.MagicMock() | |||
with mock.patch.object(sitemap_file.items, | |||
'SitemapItem') as mocked_sitemap_item: | |||
with mock.patch.object(sitemap_file, 'time'): | |||
self.spider.parse_item(response) | |||
self.assertTrue(mocked_sitemap_item.called) | |||
def test_parse_items_gets_path(self): | |||
response = mock.MagicMock() | |||
with mock.patch.object(sitemap_file.items, 'SitemapItem'): | |||
with mock.patch.object(sitemap_file.urlparse, | |||
'urlsplit') as mocked_urlsplit: | |||
with mock.patch.object(sitemap_file, 'time'): | |||
self.spider.parse_item(response) | |||
self.assertTrue(mocked_urlsplit.called) | |||
def test_parse_items_low_priority_weekly_freq(self): | |||
response = mock.MagicMock() | |||
path = sitemap_file.urlparse.SplitResult( | |||
scheme='https', | |||
netloc='docs.openstack.com', | |||
path='/kilo', | |||
query='', | |||
fragment='' | |||
) | |||
with mock.patch.object(sitemap_file.urlparse, 'urlsplit', | |||
return_value=path): | |||
with mock.patch.object(sitemap_file, 'time'): | |||
returned_item = self.spider.parse_item(response) | |||
self.assertEqual('0.5', returned_item['priority']) | |||
self.assertEqual('weekly', returned_item['changefreq']) | |||
def test_parse_items_high_priority_daily_freq(self): | |||
response = mock.MagicMock() | |||
path = sitemap_file.urlparse.SplitResult( | |||
scheme='https', | |||
netloc='docs.openstack.com', | |||
path='/mitaka', | |||
query='', | |||
fragment='' | |||
) | |||
with mock.patch.object(sitemap_file.urlparse, 'urlsplit', | |||
return_value=path): | |||
with mock.patch.object(sitemap_file, 'time'): | |||
returned_item = self.spider.parse_item(response) | |||
self.assertEqual('1.0', returned_item['priority']) | |||
self.assertEqual('daily', returned_item['changefreq']) | |||
def test_parse_returns_populated_item(self): | |||
response = mock.MagicMock() | |||
path = sitemap_file.urlparse.SplitResult( | |||
scheme='https', | |||
netloc='docs.openstack.com', | |||
path='/mitaka', | |||
query='', | |||
fragment='' | |||
) | |||
with mock.patch.object(sitemap_file.urlparse, 'urlsplit', | |||
return_value=path): | |||
with mock.patch.object(sitemap_file, 'time'): | |||
returned_item = self.spider.parse_item(response) | |||
self.assertEqual(4, len(returned_item)) | |||
if __name__ == '__main__': | |||
unittest.main() |
@ -0,0 +1,37 @@ | |||
# Licensed under the Apache License, Version 2.0 (the "License"); you may | |||
# not use this file except in compliance with the License. You may obtain | |||
# a copy of the License at | |||
# | |||
# http://www.apache.org/licenses/LICENSE-2.0 | |||
# | |||
# Unless required by applicable law or agreed to in writing, software | |||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT | |||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | |||
# License for the specific language governing permissions and limitations | |||
# under the License. | |||
import mock | |||
from sitemap.generator import items | |||
import unittest | |||
class TestSitemapItem(unittest.TestCase): | |||
def test_class_type(self): | |||
self.assertTrue(type(items.SitemapItem) is items.scrapy.item.ItemMeta) | |||
def test_class_supports_fields(self): | |||
with mock.patch.object(items.scrapy.item, 'Field'): | |||
a = items.SitemapItem() | |||
supported_fields = ['loc', 'lastmod', 'priority', 'changefreq'] | |||
for field in supported_fields: | |||
a[field] = field | |||
not_supported_fields = ['some', 'random', 'fields'] | |||
for field in not_supported_fields: | |||
with self.assertRaises(KeyError): | |||
a[field] = field | |||
if __name__ == '__main__': | |||
unittest.main() |
@ -0,0 +1,202 @@ | |||
# Licensed under the Apache License, Version 2.0 (the "License"); you may | |||
# not use this file except in compliance with the License. You may obtain | |||
# a copy of the License at | |||
# | |||
# http://www.apache.org/licenses/LICENSE-2.0 | |||
# | |||
# Unless required by applicable law or agreed to in writing, software | |||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT | |||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | |||
# License for the specific language governing permissions and limitations | |||
# under the License. | |||
import mock | |||
from sitemap.generator import pipelines | |||
import unittest | |||
class TestSitemapItemExporter(unittest.TestCase): | |||
def test_start_exporting(self): | |||
output = mock.MagicMock() | |||
itemExplorer = pipelines.SitemapItemExporter(output) | |||
with mock.patch.object(itemExplorer.xg, 'startDocument', | |||
return_value=None) as mock_start_document: | |||
with mock.patch.object(itemExplorer.xg, 'startElement', | |||
return_value=None) as mock_start_element: | |||
itemExplorer.start_exporting() | |||
self.assertTrue(mock_start_document.called) | |||
self.assertTrue(mock_start_element.called) | |||
class TestIgnoreDuplicateUrls(unittest.TestCase): | |||
def setUp(self): | |||
self.ignore_urls = pipelines.IgnoreDuplicateUrls() | |||
def test_set_is_set_at_init(self): | |||
self.assertTrue(isinstance(self.ignore_urls.processed, set)) | |||
def test_set_is_empty_at_init(self): | |||
self.assertEqual(len(self.ignore_urls.processed), 0) | |||
def test_duplicate_url(self): | |||
self.ignore_urls.processed.add('url') | |||
item = {'loc': 'url'} | |||
spider = mock.MagicMock() | |||
with self.assertRaises(pipelines.scrapy.exceptions.DropItem): | |||
self.ignore_urls.process_item(item, spider) | |||
def test_url_added_to_processed(self): | |||
self.assertFalse('url' in self.ignore_urls.processed) | |||
item = {'loc': 'url'} | |||
spider = mock.MagicMock() | |||
self.ignore_urls.process_item(item, spider) | |||
self.assertTrue('url' in self.ignore_urls.processed) | |||
def test_item_is_returned(self): | |||
item = {'loc': 'url'} | |||
spider = mock.MagicMock() | |||
returned_item = self.ignore_urls.process_item(item, spider) | |||
self.assertEqual(item, returned_item) | |||
class TestExportSitemap(unittest.TestCase): | |||
def setUp(self): | |||
self.export_sitemap = pipelines.ExportSitemap() | |||
self.spider = mock.MagicMock() | |||
def test_variables_set_at_init(self): | |||
self.assertTrue(isinstance(self.export_sitemap.files, dict)) | |||
self.assertTrue(self.export_sitemap.exporter is None) | |||
def test_spider_opened_calls_open(self): | |||
with mock.patch.object(pipelines, 'open', | |||
return_value=None) as mocked_open: | |||
with mock.patch.object(pipelines, | |||
'SitemapItemExporter'): | |||
self.export_sitemap.spider_opened(self.spider) | |||
self.assertTrue(mocked_open.called) | |||
def test_spider_opened_assigns_spider(self): | |||
prev_len = len(self.export_sitemap.files) | |||
with mock.patch.object(pipelines, 'open', | |||
return_value=None): | |||
with mock.patch.object(pipelines, | |||
'SitemapItemExporter'): | |||
self.export_sitemap.spider_opened(self.spider) | |||
after_len = len(self.export_sitemap.files) | |||
self.assertTrue(after_len - prev_len, 1) | |||
def test_spider_opened_instantiates_exporter(self): | |||
with mock.patch.object(pipelines, 'open', | |||
return_value=None): | |||
with mock.patch.object(pipelines, | |||
'SitemapItemExporter') as mocked_exporter: | |||
self.export_sitemap.spider_opened(self.spider) | |||
self.assertTrue(mocked_exporter.called) | |||
def test_spider_opened_exporter_starts_exporting(self): | |||
with mock.patch.object(pipelines, 'open', | |||
return_value=None): | |||
with mock.patch.object(pipelines.SitemapItemExporter, | |||
'start_exporting') as mocked_start: | |||
self.export_sitemap.spider_opened(self.spider) | |||
self.assertTrue(mocked_start.called) | |||
def test_spider_closed_calls_finish(self): | |||
self.export_sitemap.exporter = mock.MagicMock() | |||
self.export_sitemap.exporter.finish_exporting = mock.MagicMock() | |||
self.export_sitemap.files[self.spider] = mock.MagicMock() | |||
with mock.patch.object(pipelines, 'lxml'): | |||
with mock.patch.object(pipelines, 'open'): | |||
self.export_sitemap.spider_closed(self.spider) | |||
self.assertTrue(self.export_sitemap.exporter.finish_exporting.called) | |||
def test_spider_closed_pops_spider(self): | |||
self.export_sitemap.exporter = mock.MagicMock() | |||
self.export_sitemap.files[self.spider] = mock.MagicMock() | |||
self.assertTrue(self.spider in self.export_sitemap.files) | |||
with mock.patch.object(pipelines, 'lxml'): | |||
with mock.patch.object(pipelines, 'open'): | |||
self.export_sitemap.spider_closed(self.spider) | |||
self.assertFalse(self.spider in self.export_sitemap.files) | |||
def test_spider_closed_parses_with_lxml(self): | |||
self.export_sitemap.exporter = mock.MagicMock() | |||
self.export_sitemap.exporter.finish_exporting = mock.MagicMock() | |||
self.export_sitemap.files[self.spider] = mock.MagicMock() | |||
with mock.patch.object(pipelines.lxml, 'etree'): | |||
with mock.patch.object(pipelines.lxml.etree, | |||
'parse') as mocked_lxml_parse: | |||
with mock.patch.object(pipelines, 'open'): | |||
self.export_sitemap.spider_closed(self.spider) | |||
self.assertTrue(mocked_lxml_parse.called) | |||
def test_spider_closed_opens_xml_files(self): | |||
self.export_sitemap.exporter = mock.MagicMock() | |||
self.export_sitemap.exporter.finish_exporting = mock.MagicMock() | |||
self.export_sitemap.files[self.spider] = mock.MagicMock() | |||
with mock.patch.object(pipelines, 'lxml'): | |||
with mock.patch.object(pipelines, 'open') as mocked_open: | |||
self.export_sitemap.spider_closed(self.spider) | |||
self.assertTrue(mocked_open.called) | |||
def test_spider_closed_writes_tree(self): | |||
self.export_sitemap.exporter = mock.MagicMock() | |||
self.export_sitemap.exporter.finish_exporting = mock.MagicMock() | |||
self.export_sitemap.files[self.spider] = mock.MagicMock() | |||
with mock.patch.object(pipelines.lxml, 'etree'): | |||
with mock.patch.object(pipelines.lxml.etree, | |||
'tostring') as mocked_lxml_tostring: | |||
with mock.patch.object(pipelines, 'open'): | |||
self.export_sitemap.spider_closed(self.spider) | |||
self.assertTrue(mocked_lxml_tostring.called) | |||
def test_process_item_exports_item(self): | |||
item = spider = self.export_sitemap.exporter = mock.MagicMock() | |||
self.export_sitemap.exporter.export_item = mock.MagicMock() | |||
self.export_sitemap.process_item(item, spider) | |||
self.assertTrue(self.export_sitemap.exporter.export_item.called) | |||
def test_process_item_returns_item(self): | |||
spider = self.export_sitemap.exporter = mock.MagicMock() | |||
item = {'random': 'item'} | |||
returned_item = self.export_sitemap.process_item(item, spider) | |||
self.assertEqual(item, returned_item) | |||
def test_from_crawler_exists(self): | |||
attr_exists = hasattr(pipelines.ExportSitemap, 'from_crawler') | |||
attr_callable = callable(getattr(pipelines.ExportSitemap, | |||
'from_crawler')) | |||
self.assertTrue(attr_exists and attr_callable) | |||
def test_from_crawler_assigns_pipeline(self): | |||
crawler = mock.MagicMock() | |||
pipelines.ExportSitemap.from_crawler(crawler) | |||
# still thinking how to go about here. | |||
if __name__ == '__main__': | |||
unittest.main() |