Browse Source

Update sitemap tests

- add py3 to tox.ini (gate already tests py3)
- move all tests to $GITROOT/test so they can all run
  through testr
- add scrapy to test-requirements.txt to support sitemap tests
- move tests from test_items.py to test_sitemap_file.py
- fix broken sitemap tests
- add newton to list of old releases in sitemap_file.py
- ignore flake8 H101 as it returns false positives for Sphinx conf.py
- Use openstackdocstheme for docs
- Update sitemap README
- Restructure repo docs
- fix minor style issues

Change-Id: I22c018149b2eefde6ca5c38c22ac06886fe9a7a8
tags/1.6.0
Brian Moss 2 years ago
parent
commit
d3bc42483a

+ 1
- 0
.gitignore View File

@@ -12,6 +12,7 @@ eggs
12 12
 sdist
13 13
 
14 14
 # Unit test / coverage reports
15
+.cache
15 16
 .coverage
16 17
 .tox
17 18
 .testrepository

+ 7
- 8
README.rst View File

@@ -8,7 +8,7 @@ Team and repository tags
8 8
 .. Change things from this point on
9 9
 
10 10
 OpenStack Doc Tools
11
-*******************
11
+~~~~~~~~~~~~~~~~~~~
12 12
 
13 13
 This repository contains tools used by the OpenStack Documentation
14 14
 project.
@@ -16,8 +16,12 @@ project.
16 16
 For more details, see the `OpenStack Documentation Contributor Guide
17 17
 <http://docs.openstack.org/contributor-guide/>`_.
18 18
 
19
+* License: Apache License, Version 2.0
20
+* Source: https://git.openstack.org/cgit/openstack/openstack-doc-tools
21
+* Bugs: https://bugs.launchpad.net/openstack-doc-tools
22
+
19 23
 Prerequisites
20
-=============
24
+-------------
21 25
 
22 26
 You need to have Python 2.7 installed for using the tools.
23 27
 
@@ -57,12 +61,7 @@ On Ubuntu::
57 61
     $ apt-get install libxml2-dev libxslt-dev
58 62
 
59 63
 
60
-* License: Apache License, Version 2.0
61
-* Source: https://git.openstack.org/cgit/openstack/openstack-doc-tools
62
-* Bugs: https://bugs.launchpad.net/openstack-doc-tools
63
-
64
-
65 64
 Regenerating config option tables
66
-=================================
65
+---------------------------------
67 66
 
68 67
 See :ref:`autogenerate_config_docs`.

+ 10
- 5
doc/source/conf.py View File

@@ -14,6 +14,8 @@
14 14
 import os
15 15
 import sys
16 16
 
17
+import openstackdocstheme
18
+
17 19
 sys.path.insert(0, os.path.abspath('../..'))
18 20
 # -- General configuration ----------------------------------------------------
19 21
 
@@ -37,7 +39,7 @@ master_doc = 'index'
37 39
 
38 40
 # General information about the project.
39 41
 project = u'openstack-doc-tools'
40
-copyright = u'2014, OpenStack Foundation'
42
+copyright = u'2017, OpenStack Foundation'
41 43
 
42 44
 # If true, '()' will be appended to :func: etc. cross-reference text.
43 45
 add_function_parentheses = True
@@ -51,10 +53,13 @@ pygments_style = 'sphinx'
51 53
 
52 54
 # -- Options for HTML output --------------------------------------------------
53 55
 
54
-# The theme to use for HTML and HTML Help pages.  Major themes that come with
55
-# Sphinx are currently 'default' and 'sphinxdoc'.
56
-# html_theme_path = ["."]
57
-# html_theme = '_theme'
56
+# The theme to use for HTML and HTML Help pages.  See the documentation for
57
+# a list of builtin themes.
58
+html_theme = 'openstackdocs'
59
+
60
+# Add any paths that contain custom themes here, relative to this directory.
61
+html_theme_path = [openstackdocstheme.get_html_theme_path()]
62
+
58 63
 # html_static_path = ['static']
59 64
 
60 65
 # Output file base name for HTML help builder.

doc/source/readme.rst → doc/source/doc-tools-readme.rst View File


+ 7
- 5
doc/source/index.rst View File

@@ -1,3 +1,4 @@
1
+==============================================
1 2
 Welcome to openstack-doc-tool's documentation!
2 3
 ==============================================
3 4
 
@@ -6,16 +7,17 @@ Contents:
6 7
 .. toctree::
7 8
    :maxdepth: 2
8 9
 
9
-   readme
10
-   autogenerate_config_docs
11
-   release_notes
10
+   doc-tools-readme
12 11
    installation
13 12
    usage
13
+   autogenerate_config_docs
14
+   man/openstack-doc-test
15
+   sitemap-readme
16
+   release_notes
14 17
 
15 18
 Indices and tables
16
-==================
19
+~~~~~~~~~~~~~~~~~~
17 20
 
18 21
 * :ref:`genindex`
19 22
 * :ref:`modindex`
20 23
 * :ref:`search`
21
-

+ 9
- 5
doc/source/installation.rst View File

@@ -2,11 +2,15 @@
2 2
 Installation
3 3
 ============
4 4
 
5
-At the command line::
5
+At the command line:
6 6
 
7
-    $ pip install openstack-doc-tools
7
+   .. code-block:: console
8 8
 
9
-Or, if you have virtualenvwrapper installed::
9
+      $ pip install openstack-doc-tools
10 10
 
11
-    $ mkvirtualenv openstack-doc-tools
12
-    $ pip install openstack-doc-tools
11
+Or, if you have virtualenvwrapper installed:
12
+
13
+   .. code-block:: console
14
+
15
+      $ mkvirtualenv openstack-doc-tools
16
+      $ pip install openstack-doc-tools

+ 1
- 1
doc/source/man/openstack-doc-test.rst View File

@@ -114,4 +114,4 @@ Bugs
114 114
 
115 115
 * openstack-doc-tools is hosted on Launchpad so you can view current
116 116
   bugs at
117
-  `Bugs : openstack-manuals <https://bugs.launchpad.net/openstack-manuals/>`__
117
+  `Bugs : openstack-doc-tools <https://bugs.launchpad.net/openstack-doc-tools/>`__

+ 0
- 1
doc/source/release_notes.rst View File

@@ -1,2 +1 @@
1
-
2 1
 .. include:: ../../RELEASE_NOTES.rst

+ 1
- 0
doc/source/sitemap-readme.rst View File

@@ -0,0 +1 @@
1
+.. include:: ../../sitemap/README.rst

+ 6
- 4
doc/source/usage.rst View File

@@ -1,7 +1,9 @@
1
-========
1
+=====
2 2
 Usage
3
-========
3
+=====
4 4
 
5
-To use openstack-doc-tools in a project::
5
+To use openstack-doc-tools in a project:
6 6
 
7
-        import os_doc_tools
7
+   .. code-block:: python
8
+
9
+      import os_doc_tools

+ 61
- 27
sitemap/README.rst View File

@@ -2,46 +2,80 @@
2 2
 Sitemap Generator
3 3
 =================
4 4
 
5
-This script crawls all available sites on http://docs.openstack.org and extracts
6
-all URLs. Based on the URLs the script generates a sitemap for search engines
7
-according to the protocol described at http://www.sitemaps.org/protocol.html.
5
+This script crawls all available sites on http://docs.openstack.org and
6
+extracts all URLs. Based on the URLs the script generates a sitemap for search
7
+engines according to the `sitemaps protocol
8
+<http://www.sitemaps.org/protocol.html>`_.
8 9
 
9 10
 Installation
10
-============
11
+~~~~~~~~~~~~
11 12
 
12
-To install the needed modules you can use pip or the package management system included
13
-in your distribution. When using the package management system maybe the name of the
14
-packages differ. Installation in a virtual environment is recommended.
13
+To install the needed modules you can use pip or the package management system
14
+included in your distribution. When using the package management system maybe
15
+the name of the packages differ. Installation in a virtual environment is
16
+recommended.
15 17
 
16
-    $ virtualenv venv
17
-    $ source venv/bin/activate
18
-    $ pip install -r requirements.txt
18
+.. code-block:: console
19 19
 
20
-When using pip it's maybe necessary to install some development packages.
21
-For example on Ubuntu 16.04 install the following packages.
20
+   $ virtualenv venv
21
+   $ source venv/bin/activate
22
+   $ pip install -r requirements.txt
22 23
 
23
-    $ sudo apt install gcc libssl-dev python-dev python-virtualenv
24
+When using pip, you may also need to install some development packages. For
25
+example, on Ubuntu 16.04 install the following packages:
26
+
27
+.. code-block:: console
28
+
29
+   $ sudo apt install gcc libssl-dev python-dev python-virtualenv
24 30
 
25 31
 Usage
26
-=====
32
+~~~~~
33
+
34
+To generate a new sitemap file, change into your local clone of the
35
+``openstack/openstack-doc-tools`` repository and run the following commands:
36
+
37
+.. code-block:: console
38
+
39
+   $ cd sitemap
40
+   $ scrapy crawl sitemap
41
+
42
+The script takes several minutes to crawl all available
43
+sites on http://docs.openstack.org. The result is available in the
44
+``sitemap_docs.openstack.org.xml`` file.
45
+
46
+Options
47
+~~~~~~~
48
+
49
+domain=URL
50
+
51
+   Sets the ``domain`` to crawl. Default is ``docs.openstack.org``.
52
+
53
+   For example, to crawl http://developer.openstack.org use the following
54
+   command:
55
+
56
+   .. code-block:: console
57
+
58
+      $ scrapy crawl sitemap -a domain=developer.openstack.org
59
+
60
+   The result is available in the ``sitemap_developer.openstack.org.xml`` file.
61
+
62
+urls=URL
63
+
64
+   You can define a set of additional start URLs using the ``urls`` attribute.
65
+   Separate multiple URLs with ``,``.
27 66
 
28
-To generate a new sitemap file simply run the spider using the
29
-following command. It will take several minutes to crawl all available sites
30
-on http://docs.openstack.org. The result will be available in the file
31
-``sitemap_docs.openstack.org.xml``.
67
+   For example:
32 68
 
33
-    $ scrapy crawl sitemap
69
+   .. code-block:: console
34 70
 
35
-It's also possible to crawl other sites using the attribute ``domain``.
71
+      $ scrapy crawl sitemap -a domain=developer.openstack.org -a urls="http://developer.openstack.org/de/api-guide/quick-start/"
36 72
 
37
-For example to crawl http://developer.openstack.org use the following command.
38
-The result will be available in the file ``sitemap_developer.openstack.org.xml``.
73
+LOG_FILE=FILE
39 74
 
40
-    $ scrapy crawl sitemap -a domain=developer.openstack.org
75
+   Write log messages to the specified file.
41 76
 
42
-To write log messages into a file append the parameter ``-s LOG_FILE=scrapy.log``.
77
+   For example, to write to ``scrapy.log``:
43 78
 
44
-It is possible to define a set of additional start URLs using the attribute
45
-``urls``. Separate multiple URLs with ``,``.
79
+   .. code-block:: console
46 80
 
47
-    $ scrapy crawl sitemap -a domain=developer.openstack.org -a urls="http://developer.openstack.org/de/api-guide/quick-start/"
81
+      $ scrapy crawl sitemap -s LOG_FILE=scrapy.log

+ 2
- 2
sitemap/generator/pipelines.py View File

@@ -69,7 +69,7 @@ class ExportSitemap(object):
69 69
 
70 70
     def spider_opened(self, spider):
71 71
         output = open(os.path.join(os.getcwd(), 'sitemap_%s.xml'
72
-                      % spider.domain), 'w')
72
+                                   % spider.domain), 'w')
73 73
         self.files[spider] = output
74 74
         self.exporter = SitemapItemExporter(output, item_element='url',
75 75
                                             root_element='urlset')
@@ -80,7 +80,7 @@ class ExportSitemap(object):
80 80
         output = self.files.pop(spider)
81 81
         output.close()
82 82
         tree = lxml.etree.parse(os.path.join(os.getcwd(), "sitemap_%s.xml"
83
-                                % spider.domain))
83
+                                             % spider.domain))
84 84
         with open(os.path.join(os.getcwd(), "sitemap_%s.xml" % spider.domain),
85 85
                   'w') as pretty:
86 86
             pretty.write(lxml.etree.tostring(tree, pretty_print=True))

+ 6
- 2
sitemap/generator/spiders/sitemap_file.py View File

@@ -11,7 +11,10 @@
11 11
 # under the License.
12 12
 
13 13
 import time
14
-import urlparse
14
+try:
15
+    import urlparse
16
+except ImportError:
17
+    import urllib.parse as urlparse
15 18
 
16 19
 from scrapy import item
17 20
 from scrapy.linkextractors import LinkExtractor
@@ -41,7 +44,8 @@ class SitemapSpider(spiders.CrawlSpider):
41 44
         'juno',
42 45
         'kilo',
43 46
         'liberty',
44
-        'mitaka'
47
+        'mitaka',
48
+        'newton'
45 49
     ]])
46 50
 
47 51
     rules = [

+ 0
- 1
sitemap/requirements.txt View File

@@ -1 +0,0 @@
1
-scrapy>=1.0.0

+ 0
- 0
sitemap/test/__init__.py View File


+ 0
- 37
sitemap/test/generator/test_items.py View File

@@ -1,37 +0,0 @@
1
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
2
-# not use this file except in compliance with the License. You may obtain
3
-# a copy of the License at
4
-#
5
-#      http://www.apache.org/licenses/LICENSE-2.0
6
-#
7
-# Unless required by applicable law or agreed to in writing, software
8
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
-# License for the specific language governing permissions and limitations
11
-# under the License.
12
-
13
-import mock
14
-from sitemap.generator import items
15
-import unittest
16
-
17
-
18
-class TestSitemapItem(unittest.TestCase):
19
-
20
-    def test_class_type(self):
21
-        self.assertTrue(type(items.SitemapItem) is items.scrapy.item.ItemMeta)
22
-
23
-    def test_class_supports_fields(self):
24
-        with mock.patch.object(items.scrapy.item, 'Field'):
25
-            a = items.SitemapItem()
26
-
27
-        supported_fields = ['loc', 'lastmod', 'priority', 'changefreq']
28
-        for field in supported_fields:
29
-            a[field] = field
30
-
31
-        not_supported_fields = ['some', 'random', 'fields']
32
-        for field in not_supported_fields:
33
-            with self.assertRaises(KeyError):
34
-                a[field] = field
35
-
36
-if __name__ == '__main__':
37
-    unittest.main()

+ 1
- 1
test-requirements.txt View File

@@ -11,8 +11,8 @@ doc8 # Apache-2.0
11 11
 pylint==1.4.5 # GPLv2
12 12
 
13 13
 reno>=1.8.0 # Apache-2.0
14
+openstackdocstheme>=1.5.0 # Apache-2.0
14 15
 oslosphinx>=4.7.0 # Apache-2.0
15
-
16 16
 testrepository>=0.0.18 # Apache-2.0/BSD
17 17
 
18 18
 # mock object framework

os_doc_tools/test/__init__.py → test/__init__.py View File


os_doc_tools/test/test_index.py → test/test_index.py View File


os_doc_tools/test/test_jsoncheck.py → test/test_jsoncheck.py View File


sitemap/test/generator/test_pipelines.py → test/test_pipelines.py View File

@@ -78,26 +78,22 @@ class TestExportSitemap(unittest.TestCase):
78 78
     def test_spider_opened_calls_open(self):
79 79
         with mock.patch.object(pipelines, 'open',
80 80
                                return_value=None) as mocked_open:
81
-            with mock.patch.object(pipelines,
82
-                                   'SitemapItemExporter'):
81
+            with mock.patch.object(pipelines, 'SitemapItemExporter'):
83 82
                 self.export_sitemap.spider_opened(self.spider)
84 83
 
85 84
         self.assertTrue(mocked_open.called)
86 85
 
87 86
     def test_spider_opened_assigns_spider(self):
88 87
         prev_len = len(self.export_sitemap.files)
89
-        with mock.patch.object(pipelines, 'open',
90
-                               return_value=None):
91
-            with mock.patch.object(pipelines,
92
-                                   'SitemapItemExporter'):
88
+        with mock.patch.object(pipelines, 'open', return_value=None):
89
+            with mock.patch.object(pipelines, 'SitemapItemExporter'):
93 90
                 self.export_sitemap.spider_opened(self.spider)
94 91
 
95 92
         after_len = len(self.export_sitemap.files)
96 93
         self.assertTrue(after_len - prev_len, 1)
97 94
 
98 95
     def test_spider_opened_instantiates_exporter(self):
99
-        with mock.patch.object(pipelines, 'open',
100
-                               return_value=None):
96
+        with mock.patch.object(pipelines, 'open', return_value=None):
101 97
             with mock.patch.object(pipelines,
102 98
                                    'SitemapItemExporter') as mocked_exporter:
103 99
                 self.export_sitemap.spider_opened(self.spider)
@@ -105,8 +101,7 @@ class TestExportSitemap(unittest.TestCase):
105 101
         self.assertTrue(mocked_exporter.called)
106 102
 
107 103
     def test_spider_opened_exporter_starts_exporting(self):
108
-        with mock.patch.object(pipelines, 'open',
109
-                               return_value=None):
104
+        with mock.patch.object(pipelines, 'open', return_value=None):
110 105
             with mock.patch.object(pipelines.SitemapItemExporter,
111 106
                                    'start_exporting') as mocked_start:
112 107
                 self.export_sitemap.spider_opened(self.spider)

sitemap/test/generator/spiders/test_sitemap_file.py → test/test_sitemap_file.py View File

@@ -11,10 +11,30 @@
11 11
 # under the License.
12 12
 
13 13
 import mock
14
+import scrapy
14 15
 from sitemap.generator.spiders import sitemap_file
15 16
 import unittest
16 17
 
17 18
 
19
+class TestSitemapItem(unittest.TestCase):
20
+
21
+    def test_class_type(self):
22
+        self.assertTrue(type(sitemap_file.SitemapItem) is scrapy.item.ItemMeta)
23
+
24
+    def test_class_supports_fields(self):
25
+        with mock.patch.object(scrapy.item, 'Field'):
26
+            a = sitemap_file.SitemapItem()
27
+
28
+        supported_fields = ['loc', 'lastmod', 'priority', 'changefreq']
29
+        for field in supported_fields:
30
+            a[field] = field
31
+
32
+        not_supported_fields = ['some', 'random', 'fields']
33
+        for field in not_supported_fields:
34
+            with self.assertRaises(KeyError):
35
+                a[field] = field
36
+
37
+
18 38
 class TestSitemapSpider(unittest.TestCase):
19 39
 
20 40
     def setUp(self):
@@ -38,16 +58,18 @@ class TestSitemapSpider(unittest.TestCase):
38 58
 
39 59
     def test_parse_items_inits_sitemap(self):
40 60
         response = mock.MagicMock()
41
-        with mock.patch.object(sitemap_file.items,
61
+        with mock.patch.object(sitemap_file,
42 62
                                'SitemapItem') as mocked_sitemap_item:
43
-            with mock.patch.object(sitemap_file, 'time'):
44
-                self.spider.parse_item(response)
63
+            with mock.patch.object(sitemap_file.urlparse,
64
+                                   'urlsplit'):
65
+                with mock.patch.object(sitemap_file, 'time'):
66
+                    self.spider.parse_item(response)
45 67
 
46 68
         self.assertTrue(mocked_sitemap_item.called)
47 69
 
48 70
     def test_parse_items_gets_path(self):
49 71
         response = mock.MagicMock()
50
-        with mock.patch.object(sitemap_file.items, 'SitemapItem'):
72
+        with mock.patch.object(sitemap_file, 'SitemapItem'):
51 73
             with mock.patch.object(sitemap_file.urlparse,
52 74
                                    'urlsplit') as mocked_urlsplit:
53 75
                 with mock.patch.object(sitemap_file, 'time'):
@@ -60,7 +82,7 @@ class TestSitemapSpider(unittest.TestCase):
60 82
         path = sitemap_file.urlparse.SplitResult(
61 83
             scheme='https',
62 84
             netloc='docs.openstack.com',
63
-            path='/kilo',
85
+            path='/mitaka',
64 86
             query='',
65 87
             fragment=''
66 88
         )
@@ -77,7 +99,7 @@ class TestSitemapSpider(unittest.TestCase):
77 99
         path = sitemap_file.urlparse.SplitResult(
78 100
             scheme='https',
79 101
             netloc='docs.openstack.com',
80
-            path='/mitaka',
102
+            path='/ocata',
81 103
             query='',
82 104
             fragment=''
83 105
         )
@@ -94,7 +116,7 @@ class TestSitemapSpider(unittest.TestCase):
94 116
         path = sitemap_file.urlparse.SplitResult(
95 117
             scheme='https',
96 118
             netloc='docs.openstack.com',
97
-            path='/mitaka',
119
+            path='/ocata',
98 120
             query='',
99 121
             fragment=''
100 122
         )

+ 10
- 3
tox.ini View File

@@ -1,6 +1,6 @@
1 1
 [tox]
2 2
 minversion = 2.0
3
-envlist = py27,pep8
3
+envlist = py3,py27,pep8
4 4
 skipsdist = True
5 5
 
6 6
 [testenv]
@@ -9,7 +9,10 @@ install_command = {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:h
9 9
 setenv =
10 10
    VIRTUAL_ENV={envdir}
11 11
    CLIENT_NAME=openstack-doc-tools
12
-deps = -r{toxinidir}/test-requirements.txt
12
+# Install also sitemap scraping tool, not installed by default
13
+# therefore not in requirements file
14
+deps = scrapy>=1.0.0
15
+       -r{toxinidir}/test-requirements.txt
13 16
        -r{toxinidir}/requirements.txt
14 17
 commands = python setup.py testr --slowest --testr-args='{posargs}'
15 18
 
@@ -27,11 +30,14 @@ commands =
27 30
         cleanup/remove_trailing_whitespaces.sh
28 31
 
29 32
 [testenv:pylint]
30
-commands = pylint os_doc_tools cleanup
33
+commands = pylint os_doc_tools cleanup sitemap
31 34
 
32 35
 [testenv:releasenotes]
33 36
 commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
34 37
 
38
+[testenv:sitemap]
39
+# commands = functional test command goes here
40
+
35 41
 [testenv:venv]
36 42
 commands = {posargs}
37 43
 
@@ -44,3 +50,4 @@ builtins = _
44 50
 exclude=.venv,.git,.tox,dist,*lib/python*,*egg,build,*autogenerate_config_docs/venv,*autogenerate_config_docs/sources
45 51
 # 28 is currently the most complex thing we have
46 52
 max-complexity=29
53
+ignore = H101

Loading…
Cancel
Save