Renamed sitemap file to avoid module name conflict when importing at the sitemap unittest Added py.test tox environment Change-Id: I94480e374b29802414b62591a51c04ecd804905e Closes-Bug: #1387716changes/22/284122/34
Percila
7 years ago
committed by
Andreas Jaeger
parent
dbd8a0afa0
commit
c050836a8f
@ -0,0 +1,7 @@ |
||||
[DEFAULT] |
||||
test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ |
||||
OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ |
||||
OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ |
||||
${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION |
||||
test_id_option=--load-list $IDFILE |
||||
test_list_option=--list |
||||
@ -0,0 +1,37 @@ |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
||||
# not use this file except in compliance with the License. You may obtain |
||||
# a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
||||
# License for the specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
import mock |
||||
from os_doc_tools import index |
||||
import unittest |
||||
|
||||
|
||||
class TestGenerateIndex(unittest.TestCase): |
||||
def test_dir_created(self): |
||||
path = 'path' |
||||
with mock.patch.object(index, 'open'): |
||||
with mock.patch.object(index.os, 'mkdir') as mock_mkdir: |
||||
index.generate_index_file(path) |
||||
self.assertTrue(mock_mkdir.called) |
||||
|
||||
def test_dir_not_created_when_exists(self): |
||||
path = 'path' |
||||
with mock.patch.object(index, 'open'): |
||||
with mock.patch.object(index.os, 'mkdir') as mock_mkdir: |
||||
with mock.patch.object(index.os.path, 'isdir', |
||||
returned_value=True): |
||||
index.generate_index_file(path) |
||||
self.assertFalse(mock_mkdir.called) |
||||
|
||||
|
||||
if __name__ == '__main__': |
||||
unittest.main() |
||||
@ -0,0 +1,98 @@ |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
||||
# not use this file except in compliance with the License. You may obtain |
||||
# a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
||||
# License for the specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
import mock |
||||
from os_doc_tools import jsoncheck |
||||
import unittest |
||||
|
||||
|
||||
class MockOpen(object): |
||||
|
||||
def read(self): |
||||
return "raw" |
||||
|
||||
def write(self): |
||||
return True |
||||
|
||||
|
||||
class TestFileFunctions(unittest.TestCase): |
||||
|
||||
def test_indent_note(self): |
||||
note = "Hello\nWorld" |
||||
with mock.patch.object(jsoncheck.textwrap, 'fill') as mock_fill: |
||||
mock_fill.return_value = "Hello World" |
||||
jsoncheck._indent_note(note) |
||||
mock_fill.assert_any_call('Hello', initial_indent=' ', |
||||
subsequent_indent=' ', |
||||
width=80) |
||||
mock_fill.assert_any_call('World', initial_indent=' ', |
||||
subsequent_indent=' ', |
||||
width=80) |
||||
|
||||
def test_get_demjson_diagnostics(self): |
||||
raw = "raw" |
||||
|
||||
with mock.patch.object(jsoncheck.demjson, 'decode', return_value=True): |
||||
errstr = jsoncheck._get_demjson_diagnostics(raw) |
||||
self.assertTrue(errstr is None) |
||||
|
||||
with mock.patch.object(jsoncheck.demjson, 'decode') as mock_decode: |
||||
mock_decode.side_effect = jsoncheck.demjson.JSONError(raw) |
||||
errstr = jsoncheck._get_demjson_diagnostics(raw) |
||||
expected_error_str = " Error: raw" |
||||
self.assertEqual(errstr, expected_error_str) |
||||
|
||||
def test_parse_json(self): |
||||
raw = "raw" |
||||
with mock.patch.object(jsoncheck.json, 'loads', |
||||
return_value="Success"): |
||||
parsed = jsoncheck._parse_json(raw) |
||||
self.assertEqual(parsed, "Success") |
||||
|
||||
with mock.patch.object(jsoncheck.json, 'loads') as mock_loads: |
||||
mock_loads.side_effect = ValueError() |
||||
with self.assertRaises(jsoncheck.ParserException): |
||||
parsed = jsoncheck._parse_json(raw) |
||||
|
||||
def test_format_parsed_json(self): |
||||
with mock.patch.object(jsoncheck.json, 'dumps') as mock_dumps: |
||||
mock_dumps.return_value = "Success" |
||||
returned_value = jsoncheck._format_parsed_json('raw') |
||||
self.assertEqual(returned_value, "Success\n") |
||||
self.assertTrue(mock_dumps.called) |
||||
|
||||
def test_process_file(self): |
||||
with mock.patch.object(jsoncheck, 'open', returned_value=MockOpen()): |
||||
with mock.patch.object(jsoncheck, '_parse_json') as mock_parse: |
||||
mock_parse.side_effect = jsoncheck.ParserException |
||||
with self.assertRaises(ValueError): |
||||
jsoncheck._process_file('path') |
||||
|
||||
with mock.patch.object(jsoncheck, 'open', returned_value=MockOpen()): |
||||
with mock.patch.object(jsoncheck, '_parse_json', |
||||
returned_value="Success"): |
||||
with mock.patch.object(jsoncheck, '_format_parsed_json', |
||||
returned_value="not_raw"): |
||||
with self.assertRaises(ValueError): |
||||
jsoncheck._process_file('path', 'check') |
||||
|
||||
with mock.patch.object(jsoncheck, 'open', returned_value=MockOpen()): |
||||
with mock.patch.object(jsoncheck, '_parse_json', |
||||
returned_value="Success"): |
||||
with mock.patch.object(jsoncheck, '_format_parsed_json', |
||||
returned_value="not_raw"): |
||||
with self.assertRaises(ValueError): |
||||
jsoncheck._process_file('path', 'formatting') |
||||
|
||||
|
||||
if __name__ == '__main__': |
||||
unittest.main() |
||||
@ -0,0 +1,110 @@ |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
||||
# not use this file except in compliance with the License. You may obtain |
||||
# a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
||||
# License for the specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
import mock |
||||
from sitemap.generator.spiders import sitemap_file |
||||
import unittest |
||||
|
||||
|
||||
class TestSitemapSpider(unittest.TestCase): |
||||
|
||||
def setUp(self): |
||||
self.spider = sitemap_file.SitemapSpider() |
||||
|
||||
def test_set_vars_on_init(self): |
||||
domain = 'docs.openstack.org' |
||||
self.assertEqual(self.spider.domain, domain) |
||||
self.assertEqual(self.spider.allowed_domains, [domain]) |
||||
self.assertEqual(self.spider.start_urls, ['http://%s' % domain]) |
||||
|
||||
def test_start_urls_get_appended(self): |
||||
urls = 'new.openstack.org, old.openstack.org' |
||||
urls_len = len(urls.split(',')) |
||||
spider_len = len(self.spider.start_urls) |
||||
|
||||
spider_with_urls = sitemap_file.SitemapSpider(urls=urls) |
||||
spider_with_urls_len = len(spider_with_urls.start_urls) |
||||
|
||||
self.assertEqual(spider_with_urls_len, (urls_len + spider_len)) |
||||
|
||||
def test_parse_items_inits_sitemap(self): |
||||
response = mock.MagicMock() |
||||
with mock.patch.object(sitemap_file.items, |
||||
'SitemapItem') as mocked_sitemap_item: |
||||
with mock.patch.object(sitemap_file, 'time'): |
||||
self.spider.parse_item(response) |
||||
|
||||
self.assertTrue(mocked_sitemap_item.called) |
||||
|
||||
def test_parse_items_gets_path(self): |
||||
response = mock.MagicMock() |
||||
with mock.patch.object(sitemap_file.items, 'SitemapItem'): |
||||
with mock.patch.object(sitemap_file.urlparse, |
||||
'urlsplit') as mocked_urlsplit: |
||||
with mock.patch.object(sitemap_file, 'time'): |
||||
self.spider.parse_item(response) |
||||
|
||||
self.assertTrue(mocked_urlsplit.called) |
||||
|
||||
def test_parse_items_low_priority_weekly_freq(self): |
||||
response = mock.MagicMock() |
||||
path = sitemap_file.urlparse.SplitResult( |
||||
scheme='https', |
||||
netloc='docs.openstack.com', |
||||
path='/kilo', |
||||
query='', |
||||
fragment='' |
||||
) |
||||
with mock.patch.object(sitemap_file.urlparse, 'urlsplit', |
||||
return_value=path): |
||||
with mock.patch.object(sitemap_file, 'time'): |
||||
returned_item = self.spider.parse_item(response) |
||||
|
||||
self.assertEqual('0.5', returned_item['priority']) |
||||
self.assertEqual('weekly', returned_item['changefreq']) |
||||
|
||||
def test_parse_items_high_priority_daily_freq(self): |
||||
response = mock.MagicMock() |
||||
path = sitemap_file.urlparse.SplitResult( |
||||
scheme='https', |
||||
netloc='docs.openstack.com', |
||||
path='/mitaka', |
||||
query='', |
||||
fragment='' |
||||
) |
||||
with mock.patch.object(sitemap_file.urlparse, 'urlsplit', |
||||
return_value=path): |
||||
with mock.patch.object(sitemap_file, 'time'): |
||||
returned_item = self.spider.parse_item(response) |
||||
|
||||
self.assertEqual('1.0', returned_item['priority']) |
||||
self.assertEqual('daily', returned_item['changefreq']) |
||||
|
||||
def test_parse_returns_populated_item(self): |
||||
response = mock.MagicMock() |
||||
path = sitemap_file.urlparse.SplitResult( |
||||
scheme='https', |
||||
netloc='docs.openstack.com', |
||||
path='/mitaka', |
||||
query='', |
||||
fragment='' |
||||
) |
||||
with mock.patch.object(sitemap_file.urlparse, 'urlsplit', |
||||
return_value=path): |
||||
with mock.patch.object(sitemap_file, 'time'): |
||||
returned_item = self.spider.parse_item(response) |
||||
|
||||
self.assertEqual(4, len(returned_item)) |
||||
|
||||
|
||||
if __name__ == '__main__': |
||||
unittest.main() |
||||
@ -0,0 +1,37 @@ |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
||||
# not use this file except in compliance with the License. You may obtain |
||||
# a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
||||
# License for the specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
import mock |
||||
from sitemap.generator import items |
||||
import unittest |
||||
|
||||
|
||||
class TestSitemapItem(unittest.TestCase): |
||||
|
||||
def test_class_type(self): |
||||
self.assertTrue(type(items.SitemapItem) is items.scrapy.item.ItemMeta) |
||||
|
||||
def test_class_supports_fields(self): |
||||
with mock.patch.object(items.scrapy.item, 'Field'): |
||||
a = items.SitemapItem() |
||||
|
||||
supported_fields = ['loc', 'lastmod', 'priority', 'changefreq'] |
||||
for field in supported_fields: |
||||
a[field] = field |
||||
|
||||
not_supported_fields = ['some', 'random', 'fields'] |
||||
for field in not_supported_fields: |
||||
with self.assertRaises(KeyError): |
||||
a[field] = field |
||||
|
||||
if __name__ == '__main__': |
||||
unittest.main() |
||||
@ -0,0 +1,202 @@ |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
||||
# not use this file except in compliance with the License. You may obtain |
||||
# a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
||||
# License for the specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
import mock |
||||
from sitemap.generator import pipelines |
||||
import unittest |
||||
|
||||
|
||||
class TestSitemapItemExporter(unittest.TestCase): |
||||
|
||||
def test_start_exporting(self): |
||||
output = mock.MagicMock() |
||||
itemExplorer = pipelines.SitemapItemExporter(output) |
||||
|
||||
with mock.patch.object(itemExplorer.xg, 'startDocument', |
||||
return_value=None) as mock_start_document: |
||||
with mock.patch.object(itemExplorer.xg, 'startElement', |
||||
return_value=None) as mock_start_element: |
||||
itemExplorer.start_exporting() |
||||
|
||||
self.assertTrue(mock_start_document.called) |
||||
self.assertTrue(mock_start_element.called) |
||||
|
||||
|
||||
class TestIgnoreDuplicateUrls(unittest.TestCase): |
||||
|
||||
def setUp(self): |
||||
self.ignore_urls = pipelines.IgnoreDuplicateUrls() |
||||
|
||||
def test_set_is_set_at_init(self): |
||||
self.assertTrue(isinstance(self.ignore_urls.processed, set)) |
||||
|
||||
def test_set_is_empty_at_init(self): |
||||
self.assertEqual(len(self.ignore_urls.processed), 0) |
||||
|
||||
def test_duplicate_url(self): |
||||
self.ignore_urls.processed.add('url') |
||||
item = {'loc': 'url'} |
||||
spider = mock.MagicMock() |
||||
|
||||
with self.assertRaises(pipelines.scrapy.exceptions.DropItem): |
||||
self.ignore_urls.process_item(item, spider) |
||||
|
||||
def test_url_added_to_processed(self): |
||||
self.assertFalse('url' in self.ignore_urls.processed) |
||||
|
||||
item = {'loc': 'url'} |
||||
spider = mock.MagicMock() |
||||
self.ignore_urls.process_item(item, spider) |
||||
self.assertTrue('url' in self.ignore_urls.processed) |
||||
|
||||
def test_item_is_returned(self): |
||||
item = {'loc': 'url'} |
||||
spider = mock.MagicMock() |
||||
returned_item = self.ignore_urls.process_item(item, spider) |
||||
self.assertEqual(item, returned_item) |
||||
|
||||
|
||||
class TestExportSitemap(unittest.TestCase): |
||||
|
||||
def setUp(self): |
||||
self.export_sitemap = pipelines.ExportSitemap() |
||||
self.spider = mock.MagicMock() |
||||
|
||||
def test_variables_set_at_init(self): |
||||
self.assertTrue(isinstance(self.export_sitemap.files, dict)) |
||||
self.assertTrue(self.export_sitemap.exporter is None) |
||||
|
||||
def test_spider_opened_calls_open(self): |
||||
with mock.patch.object(pipelines, 'open', |
||||
return_value=None) as mocked_open: |
||||
with mock.patch.object(pipelines, |
||||
'SitemapItemExporter'): |
||||
self.export_sitemap.spider_opened(self.spider) |
||||
|
||||
self.assertTrue(mocked_open.called) |
||||
|
||||
def test_spider_opened_assigns_spider(self): |
||||
prev_len = len(self.export_sitemap.files) |
||||
with mock.patch.object(pipelines, 'open', |
||||
return_value=None): |
||||
with mock.patch.object(pipelines, |
||||
'SitemapItemExporter'): |
||||
self.export_sitemap.spider_opened(self.spider) |
||||
|
||||
after_len = len(self.export_sitemap.files) |
||||
self.assertTrue(after_len - prev_len, 1) |
||||
|
||||
def test_spider_opened_instantiates_exporter(self): |
||||
with mock.patch.object(pipelines, 'open', |
||||
return_value=None): |
||||
with mock.patch.object(pipelines, |
||||
'SitemapItemExporter') as mocked_exporter: |
||||
self.export_sitemap.spider_opened(self.spider) |
||||
|
||||
self.assertTrue(mocked_exporter.called) |
||||
|
||||
def test_spider_opened_exporter_starts_exporting(self): |
||||
with mock.patch.object(pipelines, 'open', |
||||
return_value=None): |
||||
with mock.patch.object(pipelines.SitemapItemExporter, |
||||
'start_exporting') as mocked_start: |
||||
self.export_sitemap.spider_opened(self.spider) |
||||
|
||||
self.assertTrue(mocked_start.called) |
||||
|
||||
def test_spider_closed_calls_finish(self): |
||||
self.export_sitemap.exporter = mock.MagicMock() |
||||
self.export_sitemap.exporter.finish_exporting = mock.MagicMock() |
||||
self.export_sitemap.files[self.spider] = mock.MagicMock() |
||||
|
||||
with mock.patch.object(pipelines, 'lxml'): |
||||
with mock.patch.object(pipelines, 'open'): |
||||
self.export_sitemap.spider_closed(self.spider) |
||||
|
||||
self.assertTrue(self.export_sitemap.exporter.finish_exporting.called) |
||||
|
||||
def test_spider_closed_pops_spider(self): |
||||
self.export_sitemap.exporter = mock.MagicMock() |
||||
self.export_sitemap.files[self.spider] = mock.MagicMock() |
||||
|
||||
self.assertTrue(self.spider in self.export_sitemap.files) |
||||
|
||||
with mock.patch.object(pipelines, 'lxml'): |
||||
with mock.patch.object(pipelines, 'open'): |
||||
self.export_sitemap.spider_closed(self.spider) |
||||
|
||||
self.assertFalse(self.spider in self.export_sitemap.files) |
||||
|
||||
def test_spider_closed_parses_with_lxml(self): |
||||
self.export_sitemap.exporter = mock.MagicMock() |
||||
self.export_sitemap.exporter.finish_exporting = mock.MagicMock() |
||||
self.export_sitemap.files[self.spider] = mock.MagicMock() |
||||
|
||||
with mock.patch.object(pipelines.lxml, 'etree'): |
||||
with mock.patch.object(pipelines.lxml.etree, |
||||
'parse') as mocked_lxml_parse: |
||||
with mock.patch.object(pipelines, 'open'): |
||||
self.export_sitemap.spider_closed(self.spider) |
||||
|
||||
self.assertTrue(mocked_lxml_parse.called) |
||||
|
||||
def test_spider_closed_opens_xml_files(self): |
||||
self.export_sitemap.exporter = mock.MagicMock() |
||||
self.export_sitemap.exporter.finish_exporting = mock.MagicMock() |
||||
self.export_sitemap.files[self.spider] = mock.MagicMock() |
||||
|
||||
with mock.patch.object(pipelines, 'lxml'): |
||||
with mock.patch.object(pipelines, 'open') as mocked_open: |
||||
self.export_sitemap.spider_closed(self.spider) |
||||
|
||||
self.assertTrue(mocked_open.called) |
||||
|
||||
def test_spider_closed_writes_tree(self): |
||||
self.export_sitemap.exporter = mock.MagicMock() |
||||
self.export_sitemap.exporter.finish_exporting = mock.MagicMock() |
||||
self.export_sitemap.files[self.spider] = mock.MagicMock() |
||||
|
||||
with mock.patch.object(pipelines.lxml, 'etree'): |
||||
with mock.patch.object(pipelines.lxml.etree, |
||||
'tostring') as mocked_lxml_tostring: |
||||
with mock.patch.object(pipelines, 'open'): |
||||
self.export_sitemap.spider_closed(self.spider) |
||||
|
||||
self.assertTrue(mocked_lxml_tostring.called) |
||||
|
||||
def test_process_item_exports_item(self): |
||||
item = spider = self.export_sitemap.exporter = mock.MagicMock() |
||||
self.export_sitemap.exporter.export_item = mock.MagicMock() |
||||
self.export_sitemap.process_item(item, spider) |
||||
|
||||
self.assertTrue(self.export_sitemap.exporter.export_item.called) |
||||
|
||||
def test_process_item_returns_item(self): |
||||
spider = self.export_sitemap.exporter = mock.MagicMock() |
||||
item = {'random': 'item'} |
||||
returned_item = self.export_sitemap.process_item(item, spider) |
||||
|
||||
self.assertEqual(item, returned_item) |
||||
|
||||
def test_from_crawler_exists(self): |
||||
attr_exists = hasattr(pipelines.ExportSitemap, 'from_crawler') |
||||
attr_callable = callable(getattr(pipelines.ExportSitemap, |
||||
'from_crawler')) |
||||
self.assertTrue(attr_exists and attr_callable) |
||||
|
||||
def test_from_crawler_assigns_pipeline(self): |
||||
crawler = mock.MagicMock() |
||||
pipelines.ExportSitemap.from_crawler(crawler) |
||||
# still thinking how to go about here. |
||||
|
||||
if __name__ == '__main__': |
||||
unittest.main() |
||||
Loading…
Reference in new issue