From c0c6becb5228bf3654678592936b9de04b011761 Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Sun, 3 May 2020 14:17:51 +0200 Subject: [PATCH] Cleanup py27 support Py27 is dead, let's switch to Python 3 fully and do a few cleanups: - Remove python 2.7 stanza from setup.py - Add requires on python >= 3.6 to setup.cfg so that pypi and pip know about the requirement - Remove obsolete sections from setup.cfg - Update classifiers - Switch to using sphinx-build and openstackdocstheme - Remove install_command from tox.ini, the default is fine, move constraints into deps - Remove babel.cfg, it's not needed anymore. - Switch to newer hacking version - Fix doc build problems - Update jobs - Fix py38 failure "RuntimeError: dictionary keys changed during iteration" in syntribos/clients/http/parser.py Change-Id: I5e9bf1901fbc2ba40cdda9e81917461de8a24f4e --- .zuul.yaml | 4 +--- babel.cfg | 3 --- doc/source/about.rst | 2 +- doc/source/conf.py | 15 ++++++++------- doc/source/man/syntribos.rst | 3 +-- doc/source/test-anatomy.rst | 2 +- setup.cfg | 30 +++++++----------------------- setup.py | 9 --------- syntribos/clients/http/parser.py | 6 ++++-- syntribos/tests/base.py | 4 ++-- test-requirements.txt | 6 +++--- tox.ini | 16 ++++++++-------- 12 files changed, 36 insertions(+), 64 deletions(-) delete mode 100644 babel.cfg diff --git a/.zuul.yaml b/.zuul.yaml index 1e1c1475..a222b3a9 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,7 +1,5 @@ - project: templates: - - openstack-python-jobs - - openstack-python35-jobs - - openstack-python36-jobs + - openstack-python3-victoria-jobs - publish-openstack-docs-pti - check-requirements diff --git a/babel.cfg b/babel.cfg deleted file mode 100644 index ea2b448f..00000000 --- a/babel.cfg +++ /dev/null @@ -1,3 +0,0 @@ -# Extraction from Python source files -[python: **.py] -encoding = utf-8 diff --git a/doc/source/about.rst b/doc/source/about.rst index 8474f615..80650725 100644 --- a/doc/source/about.rst +++ b/doc/source/about.rst @@ -134,7 +134,7 @@ User defined Test This test gives users the ability to fuzz using user defined fuzz data and provides an option to look for failure strings provided by the user. The fuzz -data needs to be provided using the config option :option:`[user_defined]`. +data needs to be provided using the config option `[user_defined]`. Example:: diff --git a/doc/source/conf.py b/doc/source/conf.py index 76ce4920..edbb3e08 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -20,7 +20,12 @@ sys.path.insert(0, os.path.abspath("../../")) # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named "sphinx.ext.*") or your custom ones. -extensions = ["sphinx.ext.autodoc", "sphinx.ext.intersphinx", "oslosphinx"] +extensions = ["sphinx.ext.autodoc", + "openstackdocstheme"] + +html_theme = 'openstackdocs' +repository_name = 'openstack/syntribos' +bug_project = 'ossa' # autodoc generation is a bit aggressive and a nuisance when doing heavy # text edit cycles. @@ -46,6 +51,8 @@ add_module_names = True # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" +exclude_patterns = ['man/syntribos.rst'] + # -- Options for man page output -------------------------------------------- # Grouping the document tree for man pages. @@ -72,9 +79,3 @@ htmlhelp_basename = "%sdoc" % project # [howto/manual]). latex_documents = [("index", "%s.tex" % project, "%s Documentation" % project, "OpenStack Foundation", "manual"), ] - -# Example configuration for intersphinx: refer to the Python standard library. -# intersphinx_mapping = {"http://docs.python.org/": None} -intersphinx_mapping = { - "requests": ("http://docs.python-requests.org/en/master", None) -} diff --git a/doc/source/man/syntribos.rst b/doc/source/man/syntribos.rst index 36b4711f..2fa4e5ec 100644 --- a/doc/source/man/syntribos.rst +++ b/doc/source/man/syntribos.rst @@ -72,8 +72,7 @@ Main Syntribos Config: file, to test on the target API Syntribos Commands: - {list_tests,run,dry_run} - Available commands + {list_tests,run,dry_run} Available commands list_tests List all available tests run Run syntribos with given config options dry_run Dry run syntribos with given config options diff --git a/doc/source/test-anatomy.rst b/doc/source/test-anatomy.rst index 2a63c12d..63bcc383 100644 --- a/doc/source/test-anatomy.rst +++ b/doc/source/test-anatomy.rst @@ -262,7 +262,7 @@ XML external entity, reflected cross-site scripting, Cross Origin Resource Sharing (CORS), SSL, Regex Denial of Service, JSON Parser Depth Limit, and User defined. -In order to run a specific test, use the :option:`-t, --test-types` +In order to run a specific test, use the `-t, --test-types` option and provide ``syntribos`` with a keyword, or keywords, to match from the test files located in ``syntribos/tests/``. diff --git a/setup.cfg b/setup.cfg index 2d6b2534..d9eda0d7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,8 +5,9 @@ description-file = README.rst license = Apache License, Version 2.0 author = OpenStack Security Group -author-email = openstack-dev@lists.openstack.org +author-email = openstack-discuss@lists.openstack.org home-page = https://docs.openstack.org/syntribos/latest +python-requires = >=3.6 classifier = Environment :: Console Intended Audience :: Information Technology @@ -16,10 +17,12 @@ classifier = Operating System :: POSIX :: Linux Operating System :: MacOS :: MacOS X Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 + Programming Language :: Python :: Implementation :: CPython + Programming Language :: Python :: 3 :: Only Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 Topic :: Security Topic :: Software Development :: Testing Topic :: Utilities @@ -31,24 +34,5 @@ console_scripts = oslo.config.opts = syntribos.config = syntribos.config:list_opts -[build_sphinx] -all_files = 1 -build-dir = doc/build -source-dir = doc/source - [files] packages = syntribos - -[compile_catalog] -directory = syntribos/locale -domain = syntribos - -[update_catalog] -domain = syntribos -output_dir = syntribos/locale -input_file = syntribos/locale/syntribos.pot - -[extract_messages] -keywords = _ gettext ngettext l_ lazy_gettext -mapping_file = babel.cfg -output_file = syntribos/locale/syntribos.pot diff --git a/setup.py b/setup.py index 566d8443..cd35c3c3 100644 --- a/setup.py +++ b/setup.py @@ -13,17 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT import setuptools -# In python < 2.7.4, a lazy loading of package `pbr` will break -# setuptools if some other modules registered functions in `atexit`. -# solution from: http://bugs.python.org/issue15881#msg170215 -try: - import multiprocessing # noqa -except ImportError: - pass - setuptools.setup( setup_requires=['pbr>=2.0.0'], pbr=True) diff --git a/syntribos/clients/http/parser.py b/syntribos/clients/http/parser.py index 1c27ef01..d73739ec 100644 --- a/syntribos/clients/http/parser.py +++ b/syntribos/clients/http/parser.py @@ -157,7 +157,8 @@ class RequestCreator(object): @classmethod def _replace_dict_variables(cls, dic): """Recursively evaluates all meta variables in a given dict.""" - for (key, value) in dic.items(): + copy_dic = copy.deepcopy(dic) + for (key, value) in copy_dic.items(): # Keys dont get fuzzed, so can handle them here match = re.search(cls.METAVAR, key) if match: @@ -456,7 +457,8 @@ class RequestHelperMixin(object): @classmethod def _run_iters_dict(cls, dic, action_field=""): """Run fuzz iterators for a dict type.""" - for key, val in dic.items(): + copy_dic = copy.deepcopy(dic) + for key, val in copy_dic.items(): dic[key] = val = cls._replace_iter(val) if isinstance(key, six.string_types): new_key = cls._replace_iter(key).replace(action_field, "") diff --git a/syntribos/tests/base.py b/syntribos/tests/base.py index d7712a30..16377ae0 100644 --- a/syntribos/tests/base.py +++ b/syntribos/tests/base.py @@ -243,8 +243,8 @@ class BaseTestCase(unittest.TestCase): failure and associates the test's metadata to it. :param defect_type: The type of vulnerability that Syntribos believes - it has found. This may be something like 500 error or DoS, regardless - tof whathe Test Type is. + it has found. This may be something like 500 error or DoS, + regard less of wha the Test Type is. :param severity: "Low", "Medium", or "High", depending on the defect :param description: Description of the defect :param confidence: The confidence of the defect diff --git a/test-requirements.txt b/test-requirements.txt index 074a7a1f..be2988bc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,17 +1,17 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. +hacking>=3.0,<3.1.0 # Apache-2.0 pylint<=2.1.0 # GPLv2 unittest2>=1.1.0 # BSD coverage!=4.4,>=4.0 # Apache-2.0 fixtures>=3.0.0 # Apache-2.0/BSD -flake8 # MIT mock>=2.0.0 # BSD python-subunit>=1.0.0 # Apache-2.0/BSD testrepository>=0.0.18 # Apache-2.0/BSD testscenarios>=0.4 # Apache-2.0/BSD testtools>=2.2.0 # MIT requests-mock>=1.2.0 # Apache-2.0 -sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD -oslosphinx>=4.7.0 # Apache-2.0 +sphinx>=2.0.0,!=2.1.0 # BSD +openstackdocstheme>=2.0.0 # Apache-2.0 beautifulsoup4>=4.6.0 # MIT diff --git a/tox.ini b/tox.ini index e554468d..6d88b786 100644 --- a/tox.ini +++ b/tox.ini @@ -1,35 +1,35 @@ [tox] -envlist=pep8,py27,py37 +envlist=pep8,py36,py37 skipsdist = True +ignore_basepython_conflict = True [testenv] +basepython = python3 usedevelop = True -install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages} setenv=VIRTUAL_ENV={envdir} -deps=-r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt +deps= + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt commands = coverage erase python setup.py testr --coverage --slowest --testr-args='{posargs}' coverage report -m [testenv:docs] -basepython = python3 commands = rm -rf doc/html doc/build rm -rf doc/source/apidoc doc/source/api - python setup.py build_sphinx + sphinx-build -W -b html doc/source doc/build/html whitelist_externals = rm [testenv:pep8] -basepython = python3 commands=flake8 {posargs} syntribos flake8 {posargs} tests {[testenv:pylint]commands} [testenv:venv] -basepython = python3 commands = {posargs} [flake8]