Cleanup py27 support

Py27 is dead, let's switch to Python 3 fully and do
a few cleanups:
- Remove python 2.7 stanza from setup.py
- Add requires on python >= 3.6 to setup.cfg so that pypi and pip
  know about the requirement
- Remove obsolete sections from setup.cfg
- Update classifiers
- Switch to using sphinx-build and openstackdocstheme
- Remove install_command from tox.ini, the default is fine,
  move constraints into deps
- Remove babel.cfg, it's not needed anymore.
- Switch to newer hacking version
- Fix doc build problems
- Update jobs
- Fix py38 failure "RuntimeError: dictionary keys changed during
  iteration" in syntribos/clients/http/parser.py

Change-Id: I5e9bf1901fbc2ba40cdda9e81917461de8a24f4e

.zuul.yaml

@@ -1,7 +1,5 @@
 - project:
     templates:
-      - openstack-python-jobs
-      - openstack-python35-jobs
-      - openstack-python36-jobs
+      - openstack-python3-victoria-jobs
       - publish-openstack-docs-pti
       - check-requirements

babel.cfg

@@ -1,3 +0,0 @@
-# Extraction from Python source files
-[python: **.py]
-encoding = utf-8

doc/source/about.rst

@@ -134,7 +134,7 @@ User defined Test
 
 This test gives users the ability to fuzz using user defined fuzz data and
 provides an option to look for failure strings provided by the user. The fuzz
-data needs to be provided using the config option :option:`[user_defined]`.
+data needs to be provided using the config option `[user_defined]`.
 
 Example::
 

doc/source/conf.py

@@ -20,7 +20,12 @@ sys.path.insert(0, os.path.abspath("../../"))
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named "sphinx.ext.*") or your custom ones.
-extensions = ["sphinx.ext.autodoc", "sphinx.ext.intersphinx", "oslosphinx"]
+extensions = ["sphinx.ext.autodoc",
+              "openstackdocstheme"]
+
+html_theme = 'openstackdocs'
+repository_name = 'openstack/syntribos'
+bug_project = 'ossa'
 
 # autodoc generation is a bit aggressive and a nuisance when doing heavy
 # text edit cycles.
@@ -46,6 +51,8 @@ add_module_names = True
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = "sphinx"
 
+exclude_patterns = ['man/syntribos.rst']
+
 # -- Options for man page output --------------------------------------------
 
 # Grouping the document tree for man pages.
@@ -72,9 +79,3 @@ htmlhelp_basename = "%sdoc" % project
 # [howto/manual]).
 latex_documents = [("index", "%s.tex" % project, "%s Documentation" % project,
                     "OpenStack Foundation", "manual"), ]
-
-# Example configuration for intersphinx: refer to the Python standard library.
-# intersphinx_mapping = {"http://docs.python.org/": None}
-intersphinx_mapping = {
-    "requests": ("http://docs.python-requests.org/en/master", None)
-}

doc/source/man/syntribos.rst

@@ -72,8 +72,7 @@ Main Syntribos Config:
                         file, to test on the target API
 
 Syntribos Commands:
-  {list_tests,run,dry_run}
-            Available commands
+    {list_tests,run,dry_run}  Available commands
     list_tests          List all available tests
     run                 Run syntribos with given config options
     dry_run             Dry run syntribos with given config options

doc/source/test-anatomy.rst

@@ -262,7 +262,7 @@ XML external entity, reflected cross-site scripting,
 Cross Origin Resource Sharing (CORS), SSL, Regex Denial of Service,
 JSON Parser Depth Limit, and User defined.
 
-In order to run a specific test, use the :option:`-t, --test-types`
+In order to run a specific test, use the `-t, --test-types`
 option and provide ``syntribos`` with a keyword, or keywords, to match from
 the test files located in ``syntribos/tests/``.
 

setup.cfg

@@ -5,8 +5,9 @@ description-file =
     README.rst
 license = Apache License, Version 2.0
 author = OpenStack Security Group
-author-email = openstack-dev@lists.openstack.org
+author-email = openstack-discuss@lists.openstack.org
 home-page = https://docs.openstack.org/syntribos/latest
+python-requires = >=3.6
 classifier =
     Environment :: Console
     Intended Audience :: Information Technology
@@ -16,10 +17,12 @@ classifier =
     Operating System :: POSIX :: Linux
     Operating System :: MacOS :: MacOS X
     Programming Language :: Python
-    Programming Language :: Python :: 2
-    Programming Language :: Python :: 2.7
+    Programming Language :: Python :: Implementation :: CPython
+    Programming Language :: Python :: 3 :: Only
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.5
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
     Topic :: Security
     Topic :: Software Development :: Testing
     Topic :: Utilities
@@ -31,24 +34,5 @@ console_scripts =
 oslo.config.opts =
     syntribos.config = syntribos.config:list_opts
 
-[build_sphinx]
-all_files = 1
-build-dir = doc/build
-source-dir = doc/source
-
 [files]
 packages = syntribos
-
-[compile_catalog]
-directory = syntribos/locale
-domain = syntribos
-
-[update_catalog]
-domain = syntribos
-output_dir = syntribos/locale
-input_file = syntribos/locale/syntribos.pot
-
-[extract_messages]
-keywords = _ gettext ngettext l_ lazy_gettext
-mapping_file = babel.cfg
-output_file = syntribos/locale/syntribos.pot

setup.py

@@ -13,17 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
 import setuptools
 
-# In python < 2.7.4, a lazy loading of package `pbr` will break
-# setuptools if some other modules registered functions in `atexit`.
-# solution from: http://bugs.python.org/issue15881#msg170215
-try:
-    import multiprocessing  # noqa
-except ImportError:
-    pass
-
 setuptools.setup(
     setup_requires=['pbr>=2.0.0'],
     pbr=True)

syntribos/clients/http/parser.py

@@ -157,7 +157,8 @@ class RequestCreator(object):
     @classmethod
     def _replace_dict_variables(cls, dic):
         """Recursively evaluates all meta variables in a given dict."""
-        for (key, value) in dic.items():
+        copy_dic = copy.deepcopy(dic)
+        for (key, value) in copy_dic.items():
             # Keys dont get fuzzed, so can handle them here
             match = re.search(cls.METAVAR, key)
             if match:
@@ -456,7 +457,8 @@ class RequestHelperMixin(object):
     @classmethod
     def _run_iters_dict(cls, dic, action_field=""):
         """Run fuzz iterators for a dict type."""
-        for key, val in dic.items():
+        copy_dic = copy.deepcopy(dic)
+        for key, val in copy_dic.items():
             dic[key] = val = cls._replace_iter(val)
             if isinstance(key, six.string_types):
                 new_key = cls._replace_iter(key).replace(action_field, "")

syntribos/tests/base.py

@@ -243,8 +243,8 @@ class BaseTestCase(unittest.TestCase):
         failure and associates the test's metadata to it.
 
         :param defect_type: The type of vulnerability that Syntribos believes
-        it has found. This may be something like 500 error or DoS, regardless
-        tof whathe Test Type is.
+            it has found. This may be something like 500 error or DoS,
+            regard less of wha the Test Type is.
         :param severity: "Low", "Medium", or "High", depending on the defect
         :param description: Description of the defect
         :param confidence: The confidence of the defect

test-requirements.txt

@@ -1,17 +1,17 @@
 # The order of packages is significant, because pip processes them in the order
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
+hacking>=3.0,<3.1.0 # Apache-2.0
 pylint<=2.1.0 # GPLv2
 unittest2>=1.1.0 # BSD
 coverage!=4.4,>=4.0 # Apache-2.0
 fixtures>=3.0.0 # Apache-2.0/BSD
-flake8 # MIT
 mock>=2.0.0 # BSD
 python-subunit>=1.0.0 # Apache-2.0/BSD
 testrepository>=0.0.18 # Apache-2.0/BSD
 testscenarios>=0.4 # Apache-2.0/BSD
 testtools>=2.2.0 # MIT
 requests-mock>=1.2.0 # Apache-2.0
-sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD
-oslosphinx>=4.7.0 # Apache-2.0
+sphinx>=2.0.0,!=2.1.0 # BSD
+openstackdocstheme>=2.0.0 # Apache-2.0
 beautifulsoup4>=4.6.0 # MIT

tox.ini

@@ -1,35 +1,35 @@
 [tox]
-envlist=pep8,py27,py37
+envlist=pep8,py36,py37
 skipsdist = True
+ignore_basepython_conflict = True
 
 [testenv]
+basepython = python3
 usedevelop = True
-install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
 setenv=VIRTUAL_ENV={envdir}
-deps=-r{toxinidir}/requirements.txt
-     -r{toxinidir}/test-requirements.txt
+deps=
+    -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/test-requirements.txt
 commands =
     coverage erase
     python setup.py testr --coverage --slowest  --testr-args='{posargs}'
     coverage report -m
 
 [testenv:docs]
-basepython = python3
 commands =
     rm -rf doc/html doc/build
     rm -rf doc/source/apidoc doc/source/api
-    python setup.py build_sphinx
+    sphinx-build -W -b html doc/source doc/build/html
 whitelist_externals =
     rm
 
 [testenv:pep8]
-basepython = python3
 commands=flake8 {posargs} syntribos
          flake8 {posargs} tests
          {[testenv:pylint]commands}
 
 [testenv:venv]
-basepython = python3
 commands = {posargs}
 
 [flake8]