doc-tools unit tests

Renamed sitemap file to avoid module name conflict
when importing at the sitemap unittest

Added py.test tox environment

Change-Id: I94480e374b29802414b62591a51c04ecd804905e
Closes-Bug: #1387716
This commit is contained in:
Percila 2016-02-24 16:17:45 +03:00 committed by Njira Perci
parent dbd8a0afa0
commit f2c24dd564
14 changed files with 571 additions and 5 deletions

5
.cache/v/cache/lastfailed vendored Normal file
View File

@ -0,0 +1,5 @@
{
"sitemap/test/generator/spiders/test_sitemap_file.py": true,
"sitemap/test/generator/test_items.py": true,
"sitemap/test/generator/test_pipelines.py": true
}

View File

@ -9,8 +9,7 @@ openstack-doc-tools style commandments
Running tests
-------------
So far there are no tests included with the package but a test suite
would be welcome!
So far there are some tests included with the package.
The openstack-indexpage tool is used while building the OpenStack
documentation repositories, test building of these repositories with

View File

View File

@ -0,0 +1,67 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from os_doc_tools.openstack.common import gettextutils
import mock
import unittest
class TestTranslatorFactory(unittest.TestCase):
def test_make_translation_func_gets_translation(self):
domain = 'domain'
tf = gettextutils.TranslatorFactory(domain)
with mock.patch.object(gettextutils.gettext,
'translation') as mock_trans:
tf._make_translation_func()
self.assertTrue(mock_trans.called)
class TestEnableLazy(unittest.TestCase):
def test_enable_lazy(self):
self.assertFalse(gettextutils.USE_LAZY)
gettextutils.enable_lazy()
self.assertTrue(gettextutils.USE_LAZY)
class TestInstall(unittest.TestCase):
def test_install(self):
domain = 'domain'
with mock.patch.object(gettextutils,
'TranslatorFactory') as mock_tf:
with mock.patch.object(gettextutils.six,
'moves'):
gettextutils.install(domain)
self.assertTrue(mock_tf.called)
class TestTranslateArgs(unittest.TestCase):
def test_translate_args(self):
msg_id = 2
message = gettextutils.Message(msg_id)
expected = msg_id
translated = gettextutils._translate_args((message))
self.assertEqual(str(translated), str(expected))
def test_translate(self):
msg_id = 2
msg = gettextutils.Message(msg_id)
exp = msg_id
transltd = gettextutils.translate(msg)
self.assertEqual(str(transltd), str(exp))
if __name__ == '__main__':
unittest.main()

View File

@ -0,0 +1,37 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from os_doc_tools import index
import unittest
class TestGenerateIndex(unittest.TestCase):
def test_dir_created(self):
path = 'path'
with mock.patch.object(index, 'open'):
with mock.patch.object(index.os, 'mkdir') as mock_mkdir:
index.generate_index_file(path)
self.assertTrue(mock_mkdir.called)
def test_dir_not_created_when_exists(self):
path = 'path'
with mock.patch.object(index, 'open'):
with mock.patch.object(index.os, 'mkdir') as mock_mkdir:
with mock.patch.object(index.os.path, 'isdir',
returned_value=True):
index.generate_index_file(path)
self.assertFalse(mock_mkdir.called)
if __name__ == '__main__':
unittest.main()

View File

@ -0,0 +1,98 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from os_doc_tools import jsoncheck
import unittest
class MockOpen(object):
def read(self):
return "raw"
def write(self):
return True
class TestFileFunctions(unittest.TestCase):
def test_indent_note(self):
note = "Hello\nWorld"
with mock.patch.object(jsoncheck.textwrap, 'fill') as mock_fill:
mock_fill.return_value = "Hello World"
jsoncheck._indent_note(note)
mock_fill.assert_any_call('Hello', initial_indent=' ',
subsequent_indent=' ',
width=80)
mock_fill.assert_any_call('World', initial_indent=' ',
subsequent_indent=' ',
width=80)
def test_get_demjson_diagnostics(self):
raw = "raw"
with mock.patch.object(jsoncheck.demjson, 'decode', return_value=True):
errstr = jsoncheck._get_demjson_diagnostics(raw)
self.assertTrue(errstr is None)
with mock.patch.object(jsoncheck.demjson, 'decode') as mock_decode:
mock_decode.side_effect = jsoncheck.demjson.JSONError(raw)
errstr = jsoncheck._get_demjson_diagnostics(raw)
expected_error_str = " Error: raw"
self.assertEqual(errstr, expected_error_str)
def test_parse_json(self):
raw = "raw"
with mock.patch.object(jsoncheck.json, 'loads',
return_value="Success"):
parsed = jsoncheck._parse_json(raw)
self.assertEqual(parsed, "Success")
with mock.patch.object(jsoncheck.json, 'loads') as mock_loads:
mock_loads.side_effect = ValueError()
with self.assertRaises(jsoncheck.ParserException):
parsed = jsoncheck._parse_json(raw)
def test_format_parsed_json(self):
with mock.patch.object(jsoncheck.json, 'dumps') as mock_dumps:
mock_dumps.return_value = "Success"
returned_value = jsoncheck._format_parsed_json('raw')
self.assertEqual(returned_value, "Success\n")
self.assertTrue(mock_dumps.called)
def test_process_file(self):
with mock.patch.object(jsoncheck, 'open', returned_value=MockOpen()):
with mock.patch.object(jsoncheck, '_parse_json') as mock_parse:
mock_parse.side_effect = jsoncheck.ParserException
with self.assertRaises(ValueError):
jsoncheck._process_file('path')
with mock.patch.object(jsoncheck, 'open', returned_value=MockOpen()):
with mock.patch.object(jsoncheck, '_parse_json',
returned_value="Success"):
with mock.patch.object(jsoncheck, '_format_parsed_json',
returned_value="not_raw"):
with self.assertRaises(ValueError):
jsoncheck._process_file('path', 'check')
with mock.patch.object(jsoncheck, 'open', returned_value=MockOpen()):
with mock.patch.object(jsoncheck, '_parse_json',
returned_value="Success"):
with mock.patch.object(jsoncheck, '_format_parsed_json',
returned_value="not_raw"):
with self.assertRaises(ValueError):
jsoncheck._process_file('path', 'formatting')
if __name__ == '__main__':
unittest.main()

0
sitemap/__init__.py Normal file
View File

View File

@ -13,9 +13,9 @@
import time
import urlparse
from generator import items
from scrapy.linkextractors import LinkExtractor
from scrapy import spiders
from sitemap.generator import items
class SitemapSpider(spiders.CrawlSpider):

0
sitemap/test/__init__.py Normal file
View File

View File

@ -0,0 +1,110 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from sitemap.generator.spiders import sitemap_file
import unittest
class TestSitemapSpider(unittest.TestCase):
def setUp(self):
self.spider = sitemap_file.SitemapSpider()
def test_set_vars_on_init(self):
domain = 'docs.openstack.org'
self.assertEqual(self.spider.domain, domain)
self.assertEqual(self.spider.allowed_domains, [domain])
self.assertEqual(self.spider.start_urls, ['http://%s' % domain])
def test_start_urls_get_appended(self):
urls = 'new.openstack.org, old.openstack.org'
urls_len = len(urls.split(','))
spider_len = len(self.spider.start_urls)
spider_with_urls = sitemap_file.SitemapSpider(urls=urls)
spider_with_urls_len = len(spider_with_urls.start_urls)
self.assertEqual(spider_with_urls_len, (urls_len + spider_len))
def test_parse_items_inits_sitemap(self):
response = mock.MagicMock()
with mock.patch.object(sitemap_file.items,
'SitemapItem') as mocked_sitemap_item:
with mock.patch.object(sitemap_file, 'time'):
self.spider.parse_item(response)
self.assertTrue(mocked_sitemap_item.called)
def test_parse_items_gets_path(self):
response = mock.MagicMock()
with mock.patch.object(sitemap_file.items, 'SitemapItem'):
with mock.patch.object(sitemap_file.urlparse,
'urlsplit') as mocked_urlsplit:
with mock.patch.object(sitemap_file, 'time'):
self.spider.parse_item(response)
self.assertTrue(mocked_urlsplit.called)
def test_parse_items_low_priority_weekly_freq(self):
response = mock.MagicMock()
path = sitemap_file.urlparse.SplitResult(
scheme='https',
netloc='docs.openstack.com',
path='/kilo',
query='',
fragment=''
)
with mock.patch.object(sitemap_file.urlparse, 'urlsplit',
return_value=path):
with mock.patch.object(sitemap_file, 'time'):
returned_item = self.spider.parse_item(response)
self.assertEqual('0.5', returned_item['priority'])
self.assertEqual('weekly', returned_item['changefreq'])
def test_parse_items_high_priority_daily_freq(self):
response = mock.MagicMock()
path = sitemap_file.urlparse.SplitResult(
scheme='https',
netloc='docs.openstack.com',
path='/mitaka',
query='',
fragment=''
)
with mock.patch.object(sitemap_file.urlparse, 'urlsplit',
return_value=path):
with mock.patch.object(sitemap_file, 'time'):
returned_item = self.spider.parse_item(response)
self.assertEqual('1.0', returned_item['priority'])
self.assertEqual('daily', returned_item['changefreq'])
def test_parse_returns_populated_item(self):
response = mock.MagicMock()
path = sitemap_file.urlparse.SplitResult(
scheme='https',
netloc='docs.openstack.com',
path='/mitaka',
query='',
fragment=''
)
with mock.patch.object(sitemap_file.urlparse, 'urlsplit',
return_value=path):
with mock.patch.object(sitemap_file, 'time'):
returned_item = self.spider.parse_item(response)
self.assertEqual(4, len(returned_item))
if __name__ == '__main__':
unittest.main()

View File

@ -0,0 +1,37 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from sitemap.generator import items
import unittest
class TestSitemapItem(unittest.TestCase):
def test_class_type(self):
self.assertTrue(type(items.SitemapItem) is items.scrapy.item.ItemMeta)
def test_class_supports_fields(self):
with mock.patch.object(items.scrapy.item, 'Field'):
a = items.SitemapItem()
supported_fields = ['loc', 'lastmod', 'priority', 'changefreq']
for field in supported_fields:
a[field] = field
not_supported_fields = ['some', 'random', 'fields']
for field in not_supported_fields:
with self.assertRaises(KeyError):
a[field] = field
if __name__ == '__main__':
unittest.main()

View File

@ -0,0 +1,202 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from sitemap.generator import pipelines
import unittest
class TestSitemapItemExporter(unittest.TestCase):
def test_start_exporting(self):
output = mock.MagicMock()
itemExplorer = pipelines.SitemapItemExporter(output)
with mock.patch.object(itemExplorer.xg, 'startDocument',
return_value=None) as mock_start_document:
with mock.patch.object(itemExplorer.xg, 'startElement',
return_value=None) as mock_start_element:
itemExplorer.start_exporting()
self.assertTrue(mock_start_document.called)
self.assertTrue(mock_start_element.called)
class TestIgnoreDuplicateUrls(unittest.TestCase):
def setUp(self):
self.ignore_urls = pipelines.IgnoreDuplicateUrls()
def test_set_is_set_at_init(self):
self.assertTrue(isinstance(self.ignore_urls.processed, set))
def test_set_is_empty_at_init(self):
self.assertEqual(len(self.ignore_urls.processed), 0)
def test_duplicate_url(self):
self.ignore_urls.processed.add('url')
item = {'loc': 'url'}
spider = mock.MagicMock()
with self.assertRaises(pipelines.scrapy.exceptions.DropItem):
self.ignore_urls.process_item(item, spider)
def test_url_added_to_processed(self):
self.assertFalse('url' in self.ignore_urls.processed)
item = {'loc': 'url'}
spider = mock.MagicMock()
self.ignore_urls.process_item(item, spider)
self.assertTrue('url' in self.ignore_urls.processed)
def test_item_is_returned(self):
item = {'loc': 'url'}
spider = mock.MagicMock()
returned_item = self.ignore_urls.process_item(item, spider)
self.assertEqual(item, returned_item)
class TestExportSitemap(unittest.TestCase):
def setUp(self):
self.export_sitemap = pipelines.ExportSitemap()
self.spider = mock.MagicMock()
def test_variables_set_at_init(self):
self.assertTrue(isinstance(self.export_sitemap.files, dict))
self.assertTrue(self.export_sitemap.exporter is None)
def test_spider_opened_calls_open(self):
with mock.patch.object(pipelines, 'open',
return_value=None) as mocked_open:
with mock.patch.object(pipelines,
'SitemapItemExporter'):
self.export_sitemap.spider_opened(self.spider)
self.assertTrue(mocked_open.called)
def test_spider_opened_assigns_spider(self):
prev_len = len(self.export_sitemap.files)
with mock.patch.object(pipelines, 'open',
return_value=None):
with mock.patch.object(pipelines,
'SitemapItemExporter'):
self.export_sitemap.spider_opened(self.spider)
after_len = len(self.export_sitemap.files)
self.assertTrue(after_len - prev_len, 1)
def test_spider_opened_instantiates_exporter(self):
with mock.patch.object(pipelines, 'open',
return_value=None):
with mock.patch.object(pipelines,
'SitemapItemExporter') as mocked_exporter:
self.export_sitemap.spider_opened(self.spider)
self.assertTrue(mocked_exporter.called)
def test_spider_opened_exporter_starts_exporting(self):
with mock.patch.object(pipelines, 'open',
return_value=None):
with mock.patch.object(pipelines.SitemapItemExporter,
'start_exporting') as mocked_start:
self.export_sitemap.spider_opened(self.spider)
self.assertTrue(mocked_start.called)
def test_spider_closed_calls_finish(self):
self.export_sitemap.exporter = mock.MagicMock()
self.export_sitemap.exporter.finish_exporting = mock.MagicMock()
self.export_sitemap.files[self.spider] = mock.MagicMock()
with mock.patch.object(pipelines, 'lxml'):
with mock.patch.object(pipelines, 'open'):
self.export_sitemap.spider_closed(self.spider)
self.assertTrue(self.export_sitemap.exporter.finish_exporting.called)
def test_spider_closed_pops_spider(self):
self.export_sitemap.exporter = mock.MagicMock()
self.export_sitemap.files[self.spider] = mock.MagicMock()
self.assertTrue(self.spider in self.export_sitemap.files)
with mock.patch.object(pipelines, 'lxml'):
with mock.patch.object(pipelines, 'open'):
self.export_sitemap.spider_closed(self.spider)
self.assertFalse(self.spider in self.export_sitemap.files)
def test_spider_closed_parses_with_lxml(self):
self.export_sitemap.exporter = mock.MagicMock()
self.export_sitemap.exporter.finish_exporting = mock.MagicMock()
self.export_sitemap.files[self.spider] = mock.MagicMock()
with mock.patch.object(pipelines.lxml, 'etree'):
with mock.patch.object(pipelines.lxml.etree,
'parse') as mocked_lxml_parse:
with mock.patch.object(pipelines, 'open'):
self.export_sitemap.spider_closed(self.spider)
self.assertTrue(mocked_lxml_parse.called)
def test_spider_closed_opens_xml_files(self):
self.export_sitemap.exporter = mock.MagicMock()
self.export_sitemap.exporter.finish_exporting = mock.MagicMock()
self.export_sitemap.files[self.spider] = mock.MagicMock()
with mock.patch.object(pipelines, 'lxml'):
with mock.patch.object(pipelines, 'open') as mocked_open:
self.export_sitemap.spider_closed(self.spider)
self.assertTrue(mocked_open.called)
def test_spider_closed_writes_tree(self):
self.export_sitemap.exporter = mock.MagicMock()
self.export_sitemap.exporter.finish_exporting = mock.MagicMock()
self.export_sitemap.files[self.spider] = mock.MagicMock()
with mock.patch.object(pipelines.lxml, 'etree'):
with mock.patch.object(pipelines.lxml.etree,
'tostring') as mocked_lxml_tostring:
with mock.patch.object(pipelines, 'open'):
self.export_sitemap.spider_closed(self.spider)
self.assertTrue(mocked_lxml_tostring.called)
def test_process_item_exports_item(self):
item = spider = self.export_sitemap.exporter = mock.MagicMock()
self.export_sitemap.exporter.export_item = mock.MagicMock()
self.export_sitemap.process_item(item, spider)
self.assertTrue(self.export_sitemap.exporter.export_item.called)
def test_process_item_returns_item(self):
spider = self.export_sitemap.exporter = mock.MagicMock()
item = {'random': 'item'}
returned_item = self.export_sitemap.process_item(item, spider)
self.assertEqual(item, returned_item)
def test_from_crawler_exists(self):
attr_exists = hasattr(pipelines.ExportSitemap, 'from_crawler')
attr_callable = callable(getattr(pipelines.ExportSitemap,
'from_crawler'))
self.assertTrue(attr_exists and attr_callable)
def test_from_crawler_assigns_pipeline(self):
crawler = mock.MagicMock()
pipelines.ExportSitemap.from_crawler(crawler)
# still thinking how to go about here.
if __name__ == '__main__':
unittest.main()

View File

@ -11,4 +11,8 @@ pylint==1.4.5 # GPLv2
reno>=1.8.0 # Apache2
oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
sphinx!=1.3b1,<1.3,>=1.2.1 # BSD
# mock object framework
mock>=1.2 # BSD
pytest

View File

@ -1,6 +1,6 @@
[tox]
minversion = 1.6
envlist = py34,py27,pep8
envlist = py27,pep8
skipsdist = True
[testenv]
@ -9,6 +9,7 @@ install_command = pip install -U {opts} {packages}
setenv =
VIRTUAL_ENV={envdir}
deps = -r{toxinidir}/test-requirements.txt
-r{toxinidir}/requirements.txt
[testenv:pep8]
commands =
@ -35,6 +36,12 @@ commands = {posargs}
[testenv:docs]
commands = python setup.py build_sphinx
[testenv:py34]
commands = py.test
[testenv:py27]
commands = py.test
[flake8]
# H803 skipped on purpose per list discussion.
# E123, E125 skipped as they are invalid PEP-8.