Browse Source
Renamed sitemap file to avoid module name conflict when importing at the sitemap unittest Added py.test tox environment Change-Id: I94480e374b29802414b62591a51c04ecd804905e Closes-Bug: #1387716changes/22/284122/34
Percila
6 years ago
committed by
Andreas Jaeger
Andreas Jaeger
16 changed files with 503 additions and 8 deletions
@ -0,0 +1,7 @@ |
|||
[DEFAULT] |
|||
test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ |
|||
OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ |
|||
OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ |
|||
${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION |
|||
test_id_option=--load-list $IDFILE |
|||
test_list_option=--list |
|||
@ -0,0 +1,37 @@ |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
import mock |
|||
from os_doc_tools import index |
|||
import unittest |
|||
|
|||
|
|||
class TestGenerateIndex(unittest.TestCase): |
|||
def test_dir_created(self): |
|||
path = 'path' |
|||
with mock.patch.object(index, 'open'): |
|||
with mock.patch.object(index.os, 'mkdir') as mock_mkdir: |
|||
index.generate_index_file(path) |
|||
self.assertTrue(mock_mkdir.called) |
|||
|
|||
def test_dir_not_created_when_exists(self): |
|||
path = 'path' |
|||
with mock.patch.object(index, 'open'): |
|||
with mock.patch.object(index.os, 'mkdir') as mock_mkdir: |
|||
with mock.patch.object(index.os.path, 'isdir', |
|||
returned_value=True): |
|||
index.generate_index_file(path) |
|||
self.assertFalse(mock_mkdir.called) |
|||
|
|||
|
|||
if __name__ == '__main__': |
|||
unittest.main() |
|||
@ -0,0 +1,98 @@ |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
import mock |
|||
from os_doc_tools import jsoncheck |
|||
import unittest |
|||
|
|||
|
|||
class MockOpen(object): |
|||
|
|||
def read(self): |
|||
return "raw" |
|||
|
|||
def write(self): |
|||
return True |
|||
|
|||
|
|||
class TestFileFunctions(unittest.TestCase): |
|||
|
|||
def test_indent_note(self): |
|||
note = "Hello\nWorld" |
|||
with mock.patch.object(jsoncheck.textwrap, 'fill') as mock_fill: |
|||
mock_fill.return_value = "Hello World" |
|||
jsoncheck._indent_note(note) |
|||
mock_fill.assert_any_call('Hello', initial_indent=' ', |
|||
subsequent_indent=' ', |
|||
width=80) |
|||
mock_fill.assert_any_call('World', initial_indent=' ', |
|||
subsequent_indent=' ', |
|||
width=80) |
|||
|
|||
def test_get_demjson_diagnostics(self): |
|||
raw = "raw" |
|||
|
|||
with mock.patch.object(jsoncheck.demjson, 'decode', return_value=True): |
|||
errstr = jsoncheck._get_demjson_diagnostics(raw) |
|||
self.assertTrue(errstr is None) |
|||
|
|||
with mock.patch.object(jsoncheck.demjson, 'decode') as mock_decode: |
|||
mock_decode.side_effect = jsoncheck.demjson.JSONError(raw) |
|||
errstr = jsoncheck._get_demjson_diagnostics(raw) |
|||
expected_error_str = " Error: raw" |
|||
self.assertEqual(errstr, expected_error_str) |
|||
|
|||
def test_parse_json(self): |
|||
raw = "raw" |
|||
with mock.patch.object(jsoncheck.json, 'loads', |
|||
return_value="Success"): |
|||
parsed = jsoncheck._parse_json(raw) |
|||
self.assertEqual(parsed, "Success") |
|||
|
|||
with mock.patch.object(jsoncheck.json, 'loads') as mock_loads: |
|||
mock_loads.side_effect = ValueError() |
|||
with self.assertRaises(jsoncheck.ParserException): |
|||
parsed = jsoncheck._parse_json(raw) |
|||
|
|||
def test_format_parsed_json(self): |
|||
with mock.patch.object(jsoncheck.json, 'dumps') as mock_dumps: |
|||
mock_dumps.return_value = "Success" |
|||
returned_value = jsoncheck._format_parsed_json('raw') |
|||
self.assertEqual(returned_value, "Success\n") |
|||
self.assertTrue(mock_dumps.called) |
|||
|
|||
def test_process_file(self): |
|||
with mock.patch.object(jsoncheck, 'open', returned_value=MockOpen()): |
|||
with mock.patch.object(jsoncheck, '_parse_json') as mock_parse: |
|||
mock_parse.side_effect = jsoncheck.ParserException |
|||
with self.assertRaises(ValueError): |
|||
jsoncheck._process_file('path') |
|||
|
|||
with mock.patch.object(jsoncheck, 'open', returned_value=MockOpen()): |
|||
with mock.patch.object(jsoncheck, '_parse_json', |
|||
returned_value="Success"): |
|||
with mock.patch.object(jsoncheck, '_format_parsed_json', |
|||
returned_value="not_raw"): |
|||
with self.assertRaises(ValueError): |
|||
jsoncheck._process_file('path', 'check') |
|||
|
|||
with mock.patch.object(jsoncheck, 'open', returned_value=MockOpen()): |
|||
with mock.patch.object(jsoncheck, '_parse_json', |
|||
returned_value="Success"): |
|||
with mock.patch.object(jsoncheck, '_format_parsed_json', |
|||
returned_value="not_raw"): |
|||
with self.assertRaises(ValueError): |
|||
jsoncheck._process_file('path', 'formatting') |
|||
|
|||
|
|||
if __name__ == '__main__': |
|||
unittest.main() |
|||
@ -0,0 +1,110 @@ |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
import mock |
|||
from sitemap.generator.spiders import sitemap_file |
|||
import unittest |
|||
|
|||
|
|||
class TestSitemapSpider(unittest.TestCase): |
|||
|
|||
def setUp(self): |
|||
self.spider = sitemap_file.SitemapSpider() |
|||
|
|||
def test_set_vars_on_init(self): |
|||
domain = 'docs.openstack.org' |
|||
self.assertEqual(self.spider.domain, domain) |
|||
self.assertEqual(self.spider.allowed_domains, [domain]) |
|||
self.assertEqual(self.spider.start_urls, ['http://%s' % domain]) |
|||
|
|||
def test_start_urls_get_appended(self): |
|||
urls = 'new.openstack.org, old.openstack.org' |
|||
urls_len = len(urls.split(',')) |
|||
spider_len = len(self.spider.start_urls) |
|||
|
|||
spider_with_urls = sitemap_file.SitemapSpider(urls=urls) |
|||
spider_with_urls_len = len(spider_with_urls.start_urls) |
|||
|
|||
self.assertEqual(spider_with_urls_len, (urls_len + spider_len)) |
|||
|
|||
def test_parse_items_inits_sitemap(self): |
|||
response = mock.MagicMock() |
|||
with mock.patch.object(sitemap_file.items, |
|||
'SitemapItem') as mocked_sitemap_item: |
|||
with mock.patch.object(sitemap_file, 'time'): |
|||
self.spider.parse_item(response) |
|||
|
|||
self.assertTrue(mocked_sitemap_item.called) |
|||
|
|||
def test_parse_items_gets_path(self): |
|||
response = mock.MagicMock() |
|||
with mock.patch.object(sitemap_file.items, 'SitemapItem'): |
|||
with mock.patch.object(sitemap_file.urlparse, |
|||
'urlsplit') as mocked_urlsplit: |
|||
with mock.patch.object(sitemap_file, 'time'): |
|||
self.spider.parse_item(response) |
|||
|
|||
self.assertTrue(mocked_urlsplit.called) |
|||
|
|||
def test_parse_items_low_priority_weekly_freq(self): |
|||
response = mock.MagicMock() |
|||
path = sitemap_file.urlparse.SplitResult( |
|||
scheme='https', |
|||
netloc='docs.openstack.com', |
|||
path='/kilo', |
|||
query='', |
|||
fragment='' |
|||
) |
|||
with mock.patch.object(sitemap_file.urlparse, 'urlsplit', |
|||
return_value=path): |
|||
with mock.patch.object(sitemap_file, 'time'): |
|||
returned_item = self.spider.parse_item(response) |
|||
|
|||
self.assertEqual('0.5', returned_item['priority']) |
|||
self.assertEqual('weekly', returned_item['changefreq']) |
|||
|
|||
def test_parse_items_high_priority_daily_freq(self): |
|||
response = mock.MagicMock() |
|||
path = sitemap_file.urlparse.SplitResult( |
|||
scheme='https', |
|||
netloc='docs.openstack.com', |
|||
path='/mitaka', |
|||
query='', |
|||
fragment='' |
|||
) |
|||
with mock.patch.object(sitemap_file.urlparse, 'urlsplit', |
|||
return_value=path): |
|||
with mock.patch.object(sitemap_file, 'time'): |
|||
returned_item = self.spider.parse_item(response) |
|||
|
|||
self.assertEqual('1.0', returned_item['priority']) |
|||
self.assertEqual('daily', returned_item['changefreq']) |
|||
|
|||
def test_parse_returns_populated_item(self): |
|||
response = mock.MagicMock() |
|||
path = sitemap_file.urlparse.SplitResult( |
|||
scheme='https', |
|||
netloc='docs.openstack.com', |
|||
path='/mitaka', |
|||
query='', |
|||
fragment='' |
|||
) |
|||
with mock.patch.object(sitemap_file.urlparse, 'urlsplit', |
|||
return_value=path): |
|||
with mock.patch.object(sitemap_file, 'time'): |
|||
returned_item = self.spider.parse_item(response) |
|||
|
|||
self.assertEqual(4, len(returned_item)) |
|||
|
|||
|
|||
if __name__ == '__main__': |
|||
unittest.main() |
|||
@ -0,0 +1,37 @@ |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
import mock |
|||
from sitemap.generator import items |
|||
import unittest |
|||
|
|||
|
|||
class TestSitemapItem(unittest.TestCase): |
|||
|
|||
def test_class_type(self): |
|||
self.assertTrue(type(items.SitemapItem) is items.scrapy.item.ItemMeta) |
|||
|
|||
def test_class_supports_fields(self): |
|||
with mock.patch.object(items.scrapy.item, 'Field'): |
|||
a = items.SitemapItem() |
|||
|
|||
supported_fields = ['loc', 'lastmod', 'priority', 'changefreq'] |
|||
for field in supported_fields: |
|||
a[field] = field |
|||
|
|||
not_supported_fields = ['some', 'random', 'fields'] |
|||
for field in not_supported_fields: |
|||
with self.assertRaises(KeyError): |
|||
a[field] = field |
|||
|
|||
if __name__ == '__main__': |
|||
unittest.main() |
|||
@ -0,0 +1,202 @@ |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
import mock |
|||
from sitemap.generator import pipelines |
|||
import unittest |
|||
|
|||
|
|||
class TestSitemapItemExporter(unittest.TestCase): |
|||
|
|||
def test_start_exporting(self): |
|||
output = mock.MagicMock() |
|||
itemExplorer = pipelines.SitemapItemExporter(output) |
|||
|
|||
with mock.patch.object(itemExplorer.xg, 'startDocument', |
|||
return_value=None) as mock_start_document: |
|||
with mock.patch.object(itemExplorer.xg, 'startElement', |
|||
return_value=None) as mock_start_element: |
|||
itemExplorer.start_exporting() |
|||
|
|||
self.assertTrue(mock_start_document.called) |
|||
self.assertTrue(mock_start_element.called) |
|||
|
|||
|
|||
class TestIgnoreDuplicateUrls(unittest.TestCase): |
|||
|
|||
def setUp(self): |
|||
self.ignore_urls = pipelines.IgnoreDuplicateUrls() |
|||
|
|||
def test_set_is_set_at_init(self): |
|||
self.assertTrue(isinstance(self.ignore_urls.processed, set)) |
|||
|
|||
def test_set_is_empty_at_init(self): |
|||
self.assertEqual(len(self.ignore_urls.processed), 0) |
|||
|
|||
def test_duplicate_url(self): |
|||
self.ignore_urls.processed.add('url') |
|||
item = {'loc': 'url'} |
|||
spider = mock.MagicMock() |
|||
|
|||
with self.assertRaises(pipelines.scrapy.exceptions.DropItem): |
|||
self.ignore_urls.process_item(item, spider) |
|||
|
|||
def test_url_added_to_processed(self): |
|||
self.assertFalse('url' in self.ignore_urls.processed) |
|||
|
|||
item = {'loc': 'url'} |
|||
spider = mock.MagicMock() |
|||
self.ignore_urls.process_item(item, spider) |
|||
self.assertTrue('url' in self.ignore_urls.processed) |
|||
|
|||
def test_item_is_returned(self): |
|||
item = {'loc': 'url'} |
|||
spider = mock.MagicMock() |
|||
returned_item = self.ignore_urls.process_item(item, spider) |
|||
self.assertEqual(item, returned_item) |
|||
|
|||
|
|||
class TestExportSitemap(unittest.TestCase): |
|||
|
|||
def setUp(self): |
|||
self.export_sitemap = pipelines.ExportSitemap() |
|||
self.spider = mock.MagicMock() |
|||
|
|||
def test_variables_set_at_init(self): |
|||
self.assertTrue(isinstance(self.export_sitemap.files, dict)) |
|||
self.assertTrue(self.export_sitemap.exporter is None) |
|||
|
|||
def test_spider_opened_calls_open(self): |
|||
with mock.patch.object(pipelines, 'open', |
|||
return_value=None) as mocked_open: |
|||
with mock.patch.object(pipelines, |
|||
'SitemapItemExporter'): |
|||
self.export_sitemap.spider_opened(self.spider) |
|||
|
|||
self.assertTrue(mocked_open.called) |
|||
|
|||
def test_spider_opened_assigns_spider(self): |
|||
prev_len = len(self.export_sitemap.files) |
|||
with mock.patch.object(pipelines, 'open', |
|||
return_value=None): |
|||
with mock.patch.object(pipelines, |
|||
'SitemapItemExporter'): |
|||
self.export_sitemap.spider_opened(self.spider) |
|||
|
|||
after_len = len(self.export_sitemap.files) |
|||
self.assertTrue(after_len - prev_len, 1) |
|||
|
|||
def test_spider_opened_instantiates_exporter(self): |
|||
with mock.patch.object(pipelines, 'open', |
|||
return_value=None): |
|||
with mock.patch.object(pipelines, |
|||
'SitemapItemExporter') as mocked_exporter: |
|||
self.export_sitemap.spider_opened(self.spider) |
|||
|
|||
self.assertTrue(mocked_exporter.called) |
|||
|
|||
def test_spider_opened_exporter_starts_exporting(self): |
|||
with mock.patch.object(pipelines, 'open', |
|||
return_value=None): |
|||
with mock.patch.object(pipelines.SitemapItemExporter, |
|||
'start_exporting') as mocked_start: |
|||
self.export_sitemap.spider_opened(self.spider) |
|||
|
|||
self.assertTrue(mocked_start.called) |
|||
|
|||
def test_spider_closed_calls_finish(self): |
|||
self.export_sitemap.exporter = mock.MagicMock() |
|||
self.export_sitemap.exporter.finish_exporting = mock.MagicMock() |
|||
self.export_sitemap.files[self.spider] = mock.MagicMock() |
|||
|
|||
with mock.patch.object(pipelines, 'lxml'): |
|||
with mock.patch.object(pipelines, 'open'): |
|||
self.export_sitemap.spider_closed(self.spider) |
|||
|
|||
self.assertTrue(self.export_sitemap.exporter.finish_exporting.called) |
|||
|
|||
def test_spider_closed_pops_spider(self): |
|||
self.export_sitemap.exporter = mock.MagicMock() |
|||
self.export_sitemap.files[self.spider] = mock.MagicMock() |
|||
|
|||
self.assertTrue(self.spider in self.export_sitemap.files) |
|||
|
|||
with mock.patch.object(pipelines, 'lxml'): |
|||
with mock.patch.object(pipelines, 'open'): |
|||
self.export_sitemap.spider_closed(self.spider) |
|||
|
|||
self.assertFalse(self.spider in self.export_sitemap.files) |
|||
|
|||
def test_spider_closed_parses_with_lxml(self): |
|||
self.export_sitemap.exporter = mock.MagicMock() |
|||
self.export_sitemap.exporter.finish_exporting = mock.MagicMock() |
|||
self.export_sitemap.files[self.spider] = mock.MagicMock() |
|||
|
|||
with mock.patch.object(pipelines.lxml, 'etree'): |
|||
with mock.patch.object(pipelines.lxml.etree, |
|||
'parse') as mocked_lxml_parse: |
|||
with mock.patch.object(pipelines, 'open'): |
|||
self.export_sitemap.spider_closed(self.spider) |
|||
|
|||
self.assertTrue(mocked_lxml_parse.called) |
|||
|
|||
def test_spider_closed_opens_xml_files(self): |
|||
self.export_sitemap.exporter = mock.MagicMock() |
|||
self.export_sitemap.exporter.finish_exporting = mock.MagicMock() |
|||
self.export_sitemap.files[self.spider] = mock.MagicMock() |
|||
|
|||
with mock.patch.object(pipelines, 'lxml'): |
|||
with mock.patch.object(pipelines, 'open') as mocked_open: |
|||
self.export_sitemap.spider_closed(self.spider) |
|||
|
|||
self.assertTrue(mocked_open.called) |
|||
|
|||
def test_spider_closed_writes_tree(self): |
|||
self.export_sitemap.exporter = mock.MagicMock() |
|||
self.export_sitemap.exporter.finish_exporting = mock.MagicMock() |
|||
self.export_sitemap.files[self.spider] = mock.MagicMock() |
|||
|
|||
with mock.patch.object(pipelines.lxml, 'etree'): |
|||
with mock.patch.object(pipelines.lxml.etree, |
|||
'tostring') as mocked_lxml_tostring: |
|||
with mock.patch.object(pipelines, 'open'): |
|||
self.export_sitemap.spider_closed(self.spider) |
|||
|
|||
self.assertTrue(mocked_lxml_tostring.called) |
|||
|
|||
def test_process_item_exports_item(self): |
|||
item = spider = self.export_sitemap.exporter = mock.MagicMock() |
|||
self.export_sitemap.exporter.export_item = mock.MagicMock() |
|||
self.export_sitemap.process_item(item, spider) |
|||
|
|||
self.assertTrue(self.export_sitemap.exporter.export_item.called) |
|||
|
|||
def test_process_item_returns_item(self): |
|||
spider = self.export_sitemap.exporter = mock.MagicMock() |
|||
item = {'random': 'item'} |
|||
returned_item = self.export_sitemap.process_item(item, spider) |
|||
|
|||
self.assertEqual(item, returned_item) |
|||
|
|||
def test_from_crawler_exists(self): |
|||
attr_exists = hasattr(pipelines.ExportSitemap, 'from_crawler') |
|||
attr_callable = callable(getattr(pipelines.ExportSitemap, |
|||
'from_crawler')) |
|||
self.assertTrue(attr_exists and attr_callable) |
|||
|
|||
def test_from_crawler_assigns_pipeline(self): |
|||
crawler = mock.MagicMock() |
|||
pipelines.ExportSitemap.from_crawler(crawler) |
|||
# still thinking how to go about here. |
|||
|
|||
if __name__ == '__main__': |
|||
unittest.main() |
|||
Loading…
Reference in new issue