Tools used by OpenStack Documentation
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

203 lines
7.9 KiB

  1. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  2. # not use this file except in compliance with the License. You may obtain
  3. # a copy of the License at
  4. #
  5. # http://www.apache.org/licenses/LICENSE-2.0
  6. #
  7. # Unless required by applicable law or agreed to in writing, software
  8. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  9. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  10. # License for the specific language governing permissions and limitations
  11. # under the License.
  12. import mock
  13. from sitemap.generator import pipelines
  14. import unittest
  15. class TestSitemapItemExporter(unittest.TestCase):
  16. def test_start_exporting(self):
  17. output = mock.MagicMock()
  18. itemExplorer = pipelines.SitemapItemExporter(output)
  19. with mock.patch.object(itemExplorer.xg, 'startDocument',
  20. return_value=None) as mock_start_document:
  21. with mock.patch.object(itemExplorer.xg, 'startElement',
  22. return_value=None) as mock_start_element:
  23. itemExplorer.start_exporting()
  24. self.assertTrue(mock_start_document.called)
  25. self.assertTrue(mock_start_element.called)
  26. class TestIgnoreDuplicateUrls(unittest.TestCase):
  27. def setUp(self):
  28. self.ignore_urls = pipelines.IgnoreDuplicateUrls()
  29. def test_set_is_set_at_init(self):
  30. self.assertTrue(isinstance(self.ignore_urls.processed, set))
  31. def test_set_is_empty_at_init(self):
  32. self.assertEqual(len(self.ignore_urls.processed), 0)
  33. def test_duplicate_url(self):
  34. self.ignore_urls.processed.add('url')
  35. item = {'loc': 'url'}
  36. spider = mock.MagicMock()
  37. with self.assertRaises(pipelines.scrapy.exceptions.DropItem):
  38. self.ignore_urls.process_item(item, spider)
  39. def test_url_added_to_processed(self):
  40. self.assertFalse('url' in self.ignore_urls.processed)
  41. item = {'loc': 'url'}
  42. spider = mock.MagicMock()
  43. self.ignore_urls.process_item(item, spider)
  44. self.assertTrue('url' in self.ignore_urls.processed)
  45. def test_item_is_returned(self):
  46. item = {'loc': 'url'}
  47. spider = mock.MagicMock()
  48. returned_item = self.ignore_urls.process_item(item, spider)
  49. self.assertEqual(item, returned_item)
  50. class TestExportSitemap(unittest.TestCase):
  51. def setUp(self):
  52. self.export_sitemap = pipelines.ExportSitemap()
  53. self.spider = mock.MagicMock()
  54. def test_variables_set_at_init(self):
  55. self.assertTrue(isinstance(self.export_sitemap.files, dict))
  56. self.assertTrue(self.export_sitemap.exporter is None)
  57. def test_spider_opened_calls_open(self):
  58. with mock.patch.object(pipelines, 'open',
  59. return_value=None) as mocked_open:
  60. with mock.patch.object(pipelines,
  61. 'SitemapItemExporter'):
  62. self.export_sitemap.spider_opened(self.spider)
  63. self.assertTrue(mocked_open.called)
  64. def test_spider_opened_assigns_spider(self):
  65. prev_len = len(self.export_sitemap.files)
  66. with mock.patch.object(pipelines, 'open',
  67. return_value=None):
  68. with mock.patch.object(pipelines,
  69. 'SitemapItemExporter'):
  70. self.export_sitemap.spider_opened(self.spider)
  71. after_len = len(self.export_sitemap.files)
  72. self.assertTrue(after_len - prev_len, 1)
  73. def test_spider_opened_instantiates_exporter(self):
  74. with mock.patch.object(pipelines, 'open',
  75. return_value=None):
  76. with mock.patch.object(pipelines,
  77. 'SitemapItemExporter') as mocked_exporter:
  78. self.export_sitemap.spider_opened(self.spider)
  79. self.assertTrue(mocked_exporter.called)
  80. def test_spider_opened_exporter_starts_exporting(self):
  81. with mock.patch.object(pipelines, 'open',
  82. return_value=None):
  83. with mock.patch.object(pipelines.SitemapItemExporter,
  84. 'start_exporting') as mocked_start:
  85. self.export_sitemap.spider_opened(self.spider)
  86. self.assertTrue(mocked_start.called)
  87. def test_spider_closed_calls_finish(self):
  88. self.export_sitemap.exporter = mock.MagicMock()
  89. self.export_sitemap.exporter.finish_exporting = mock.MagicMock()
  90. self.export_sitemap.files[self.spider] = mock.MagicMock()
  91. with mock.patch.object(pipelines, 'lxml'):
  92. with mock.patch.object(pipelines, 'open'):
  93. self.export_sitemap.spider_closed(self.spider)
  94. self.assertTrue(self.export_sitemap.exporter.finish_exporting.called)
  95. def test_spider_closed_pops_spider(self):
  96. self.export_sitemap.exporter = mock.MagicMock()
  97. self.export_sitemap.files[self.spider] = mock.MagicMock()
  98. self.assertTrue(self.spider in self.export_sitemap.files)
  99. with mock.patch.object(pipelines, 'lxml'):
  100. with mock.patch.object(pipelines, 'open'):
  101. self.export_sitemap.spider_closed(self.spider)
  102. self.assertFalse(self.spider in self.export_sitemap.files)
  103. def test_spider_closed_parses_with_lxml(self):
  104. self.export_sitemap.exporter = mock.MagicMock()
  105. self.export_sitemap.exporter.finish_exporting = mock.MagicMock()
  106. self.export_sitemap.files[self.spider] = mock.MagicMock()
  107. with mock.patch.object(pipelines.lxml, 'etree'):
  108. with mock.patch.object(pipelines.lxml.etree,
  109. 'parse') as mocked_lxml_parse:
  110. with mock.patch.object(pipelines, 'open'):
  111. self.export_sitemap.spider_closed(self.spider)
  112. self.assertTrue(mocked_lxml_parse.called)
  113. def test_spider_closed_opens_xml_files(self):
  114. self.export_sitemap.exporter = mock.MagicMock()
  115. self.export_sitemap.exporter.finish_exporting = mock.MagicMock()
  116. self.export_sitemap.files[self.spider] = mock.MagicMock()
  117. with mock.patch.object(pipelines, 'lxml'):
  118. with mock.patch.object(pipelines, 'open') as mocked_open:
  119. self.export_sitemap.spider_closed(self.spider)
  120. self.assertTrue(mocked_open.called)
  121. def test_spider_closed_writes_tree(self):
  122. self.export_sitemap.exporter = mock.MagicMock()
  123. self.export_sitemap.exporter.finish_exporting = mock.MagicMock()
  124. self.export_sitemap.files[self.spider] = mock.MagicMock()
  125. with mock.patch.object(pipelines.lxml, 'etree'):
  126. with mock.patch.object(pipelines.lxml.etree,
  127. 'tostring') as mocked_lxml_tostring:
  128. with mock.patch.object(pipelines, 'open'):
  129. self.export_sitemap.spider_closed(self.spider)
  130. self.assertTrue(mocked_lxml_tostring.called)
  131. def test_process_item_exports_item(self):
  132. item = spider = self.export_sitemap.exporter = mock.MagicMock()
  133. self.export_sitemap.exporter.export_item = mock.MagicMock()
  134. self.export_sitemap.process_item(item, spider)
  135. self.assertTrue(self.export_sitemap.exporter.export_item.called)
  136. def test_process_item_returns_item(self):
  137. spider = self.export_sitemap.exporter = mock.MagicMock()
  138. item = {'random': 'item'}
  139. returned_item = self.export_sitemap.process_item(item, spider)
  140. self.assertEqual(item, returned_item)
  141. def test_from_crawler_exists(self):
  142. attr_exists = hasattr(pipelines.ExportSitemap, 'from_crawler')
  143. attr_callable = callable(getattr(pipelines.ExportSitemap,
  144. 'from_crawler'))
  145. self.assertTrue(attr_exists and attr_callable)
  146. def test_from_crawler_assigns_pipeline(self):
  147. crawler = mock.MagicMock()
  148. pipelines.ExportSitemap.from_crawler(crawler)
  149. # still thinking how to go about here.
  150. if __name__ == '__main__':
  151. unittest.main()