Tools used by OpenStack Documentation
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

198 lines
7.7 KiB

  1. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  2. # not use this file except in compliance with the License. You may obtain
  3. # a copy of the License at
  4. #
  5. # http://www.apache.org/licenses/LICENSE-2.0
  6. #
  7. # Unless required by applicable law or agreed to in writing, software
  8. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  9. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  10. # License for the specific language governing permissions and limitations
  11. # under the License.
  12. import mock
  13. from sitemap.generator import pipelines
  14. import unittest
  15. class TestSitemapItemExporter(unittest.TestCase):
  16. def test_start_exporting(self):
  17. output = mock.MagicMock()
  18. itemExplorer = pipelines.SitemapItemExporter(output)
  19. with mock.patch.object(itemExplorer.xg, 'startDocument',
  20. return_value=None) as mock_start_document:
  21. with mock.patch.object(itemExplorer.xg, 'startElement',
  22. return_value=None) as mock_start_element:
  23. itemExplorer.start_exporting()
  24. self.assertTrue(mock_start_document.called)
  25. self.assertTrue(mock_start_element.called)
  26. class TestIgnoreDuplicateUrls(unittest.TestCase):
  27. def setUp(self):
  28. self.ignore_urls = pipelines.IgnoreDuplicateUrls()
  29. def test_set_is_set_at_init(self):
  30. self.assertIsInstance(self.ignore_urls.processed, set)
  31. def test_set_is_empty_at_init(self):
  32. self.assertEqual(len(self.ignore_urls.processed), 0)
  33. def test_duplicate_url(self):
  34. self.ignore_urls.processed.add('url')
  35. item = {'loc': 'url'}
  36. spider = mock.MagicMock()
  37. with self.assertRaises(pipelines.scrapy.exceptions.DropItem):
  38. self.ignore_urls.process_item(item, spider)
  39. def test_url_added_to_processed(self):
  40. self.assertFalse('url' in self.ignore_urls.processed)
  41. item = {'loc': 'url'}
  42. spider = mock.MagicMock()
  43. self.ignore_urls.process_item(item, spider)
  44. self.assertTrue('url' in self.ignore_urls.processed)
  45. def test_item_is_returned(self):
  46. item = {'loc': 'url'}
  47. spider = mock.MagicMock()
  48. returned_item = self.ignore_urls.process_item(item, spider)
  49. self.assertEqual(item, returned_item)
  50. class TestExportSitemap(unittest.TestCase):
  51. def setUp(self):
  52. self.export_sitemap = pipelines.ExportSitemap()
  53. self.spider = mock.MagicMock()
  54. def test_variables_set_at_init(self):
  55. self.assertIsInstance(self.export_sitemap.files, dict)
  56. self.assertTrue(self.export_sitemap.exporter is None)
  57. def test_spider_opened_calls_open(self):
  58. with mock.patch.object(pipelines, 'open',
  59. return_value=None) as mocked_open:
  60. with mock.patch.object(pipelines, 'SitemapItemExporter'):
  61. self.export_sitemap.spider_opened(self.spider)
  62. self.assertTrue(mocked_open.called)
  63. def test_spider_opened_assigns_spider(self):
  64. prev_len = len(self.export_sitemap.files)
  65. with mock.patch.object(pipelines, 'open', return_value=None):
  66. with mock.patch.object(pipelines, 'SitemapItemExporter'):
  67. self.export_sitemap.spider_opened(self.spider)
  68. after_len = len(self.export_sitemap.files)
  69. self.assertTrue(after_len - prev_len, 1)
  70. def test_spider_opened_instantiates_exporter(self):
  71. with mock.patch.object(pipelines, 'open', return_value=None):
  72. with mock.patch.object(pipelines,
  73. 'SitemapItemExporter') as mocked_exporter:
  74. self.export_sitemap.spider_opened(self.spider)
  75. self.assertTrue(mocked_exporter.called)
  76. def test_spider_opened_exporter_starts_exporting(self):
  77. with mock.patch.object(pipelines, 'open', return_value=None):
  78. with mock.patch.object(pipelines.SitemapItemExporter,
  79. 'start_exporting') as mocked_start:
  80. self.export_sitemap.spider_opened(self.spider)
  81. self.assertTrue(mocked_start.called)
  82. def test_spider_closed_calls_finish(self):
  83. self.export_sitemap.exporter = mock.MagicMock()
  84. self.export_sitemap.exporter.finish_exporting = mock.MagicMock()
  85. self.export_sitemap.files[self.spider] = mock.MagicMock()
  86. with mock.patch.object(pipelines, 'lxml'):
  87. with mock.patch.object(pipelines, 'open'):
  88. self.export_sitemap.spider_closed(self.spider)
  89. self.assertTrue(self.export_sitemap.exporter.finish_exporting.called)
  90. def test_spider_closed_pops_spider(self):
  91. self.export_sitemap.exporter = mock.MagicMock()
  92. self.export_sitemap.files[self.spider] = mock.MagicMock()
  93. self.assertTrue(self.spider in self.export_sitemap.files)
  94. with mock.patch.object(pipelines, 'lxml'):
  95. with mock.patch.object(pipelines, 'open'):
  96. self.export_sitemap.spider_closed(self.spider)
  97. self.assertFalse(self.spider in self.export_sitemap.files)
  98. def test_spider_closed_parses_with_lxml(self):
  99. self.export_sitemap.exporter = mock.MagicMock()
  100. self.export_sitemap.exporter.finish_exporting = mock.MagicMock()
  101. self.export_sitemap.files[self.spider] = mock.MagicMock()
  102. with mock.patch.object(pipelines.lxml, 'etree'):
  103. with mock.patch.object(pipelines.lxml.etree,
  104. 'parse') as mocked_lxml_parse:
  105. with mock.patch.object(pipelines, 'open'):
  106. self.export_sitemap.spider_closed(self.spider)
  107. self.assertTrue(mocked_lxml_parse.called)
  108. def test_spider_closed_opens_xml_files(self):
  109. self.export_sitemap.exporter = mock.MagicMock()
  110. self.export_sitemap.exporter.finish_exporting = mock.MagicMock()
  111. self.export_sitemap.files[self.spider] = mock.MagicMock()
  112. with mock.patch.object(pipelines, 'lxml'):
  113. with mock.patch.object(pipelines, 'open') as mocked_open:
  114. self.export_sitemap.spider_closed(self.spider)
  115. self.assertTrue(mocked_open.called)
  116. def test_spider_closed_writes_tree(self):
  117. self.export_sitemap.exporter = mock.MagicMock()
  118. self.export_sitemap.exporter.finish_exporting = mock.MagicMock()
  119. self.export_sitemap.files[self.spider] = mock.MagicMock()
  120. with mock.patch.object(pipelines.lxml, 'etree'):
  121. with mock.patch.object(pipelines.lxml.etree,
  122. 'tostring') as mocked_lxml_tostring:
  123. with mock.patch.object(pipelines, 'open'):
  124. self.export_sitemap.spider_closed(self.spider)
  125. self.assertTrue(mocked_lxml_tostring.called)
  126. def test_process_item_exports_item(self):
  127. item = spider = self.export_sitemap.exporter = mock.MagicMock()
  128. self.export_sitemap.exporter.export_item = mock.MagicMock()
  129. self.export_sitemap.process_item(item, spider)
  130. self.assertTrue(self.export_sitemap.exporter.export_item.called)
  131. def test_process_item_returns_item(self):
  132. spider = self.export_sitemap.exporter = mock.MagicMock()
  133. item = {'random': 'item'}
  134. returned_item = self.export_sitemap.process_item(item, spider)
  135. self.assertEqual(item, returned_item)
  136. def test_from_crawler_exists(self):
  137. attr_exists = hasattr(pipelines.ExportSitemap, 'from_crawler')
  138. attr_callable = callable(getattr(pipelines.ExportSitemap,
  139. 'from_crawler'))
  140. self.assertTrue(attr_exists and attr_callable)
  141. def test_from_crawler_assigns_pipeline(self):
  142. crawler = mock.MagicMock()
  143. pipelines.ExportSitemap.from_crawler(crawler)
  144. # still thinking how to go about here.
  145. if __name__ == '__main__':
  146. unittest.main()