Python3 General Updates

Removed bash-like python environment statements
and fixed common error reporting to send errors
to console logs and api calls correctly. also
removes old tox.ini files which are unneeded
after service collation from previous iteration
of this project.

Change-Id: I1727c317b8fa0e2abd8a959321f4366fdecb82c8

orm/cmd/db_sync.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright (c) 2018 OpenStack Foundation
 # All Rights Reserved.
 #

orm/common/client/audit/tox.ini

@@ -1,22 +0,0 @@
-[tox]
-envlist = py27, cover, pep8
-
-[testenv]
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
-install_command = pip install -U {opts} {packages}
-
-commands = python setup.py testr
-
-[testenv:cover]
-#omitting rds/api/app.py and rds/examples/api/functional_test.py
-#since they have no need for unit test
-commands = 
-	python setup.py testr --slowest --coverage --omit=audit_client/examples/*
-        coverage report --omit=audit_client/examples/*
-
-[testenv:pep8]
-#cannot handle and 'H102  Apache 2.0 license header not found' and
-#'H202  assertRaises Exception too broad'
-#since it requires business code changes
-commands = flake8

orm/common/client/keystone/tox.ini

@@ -1,24 +0,0 @@
-[tox]
-envlist = py27,cover
-skipsdist = True
-
-[testenv]
-install_command =
-#                  constraints: {[testenv:common-constraints]install_command}
-                  pip install -U --force-reinstall {opts} {packages}
-setenv = VIRTUAL_ENV={envdir}
-         OS_TEST_PATH=./keystone_utils/tests/unit
-         PYTHONPATH = {toxinidir}/mock_keystone/:/usr/local/lib/python2.7/dist-packages/
-deps = -r{toxinidir}/test-requirements.txt
-       -r{toxinidir}/requirements.txt
-
-[testenv:cover]
-commands =
-  coverage erase
-  python setup.py testr --coverage
-  coverage report --omit="keystone_utils/tests/*"
-  coverage html --omit="keystone_utils/tests/*"
-
-[testenv:pep8]
-commands=
-    py.test --pep8 -m pep8

orm/common/orm_common/hooks/api_error_hook.py

@@ -27,15 +27,15 @@ class APIErrorHook(PecanHook):
                 result_json = err_utils.get_error_dict(401,
                                                        transaction_id,
                                                        None)
-
             else:
                 dict_body = None
                 try:
                     logger.debug('error: {}'.format(state.response))
                     dict_body = json.loads(state.response.body)
+                    result_json = dict_body
+
                     if 'line' in str(state.response.body) and 'column' in str(
                             state.response.body):
-                        result_json = dict_body
                         status_code = 400
                         if 'faultstring' in dict_body:
                             result_json = err_utils.get_error_dict(status_code,
@@ -43,14 +43,17 @@ class APIErrorHook(PecanHook):
                                                                    dict_body['faultstring'],
                                                                    "")
                     else:
-                        result_json = json.loads(dict_body['faultstring'])
+                        logger.info('Received faultstring: {}'.format(dict_body['faultstring']))
-                        logger.debug('Received faultstring: {}'.format(result_json))
                     # make sure status code in header and in body are the same
                     if 'code' in result_json:
                         status_code = result_json['code']
 
                     logger.info('Received status code: {}, transaction_id: {}, tracking_id: {}'.
                                 format(status_code, transaction_id, tracking_id))
+                    result_json = err_utils.get_error_dict(status_code,
+                                                           transaction_id,
+                                                           dict_body['faultstring'],
+                                                           "")
 
                 except ValueError:
                     msg = 'Could not read faultstring from response body!'
@@ -64,7 +67,6 @@ class APIErrorHook(PecanHook):
                                                            transaction_id,
                                                            msg,
                                                            "")
-
+        state.response.text = json.dumps(result_json)
-            setattr(state.response, 'body', json.dumps(result_json))
+        state.response.status_code = status_code
-            state.response.status_code = status_code
+        state.response.headers.add('X-RANGER-Request-Id', tracking_id)
-            state.response.headers.add('X-RANGER-Request-Id', tracking_id)

orm/orm_client/flavorgen/flavorator.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 import argparse
 import json
 import os

orm/orm_client/flavorgen/make_flav.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 import json
 import sys
 

orm/orm_client/flavorgen/make_flavor.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 import json
 import re
 import sys

orm/orm_client/flavorgen/regionator.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 import argparse
 import ast
 import json

orm/orm_client/imagegen/imageator.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 import argparse
 import json
 import os

orm/orm_client/imagegen/regionator.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 import argparse
 import ast
 import json

orm/orm_client/ormcli/imscli.py

@@ -1,4 +1,3 @@
-#!/usr/bin/python
 import argparse
 import orm.base_config as base_config
 import os

orm/orm_client/ormcli/orm

@@ -1,5 +1,4 @@
-#!/usr/bin/env python
+#! /usr/bin/python3
-
 import sys
 import ormcli
 

orm/orm_client/ormcli/ormcli.py

@@ -1,4 +1,3 @@
-#!/usr/bin/python
 import argparse
 from orm.orm_client.ormcli import cmscli
 from orm.orm_client.ormcli import fmscli

orm/orm_client/ormcli/rmscli.py

@@ -1,4 +1,3 @@
-#!/usr/bin/python
 import argparse
 import orm.base_config as base_config
 import os

orm/services/audit_trail_manager/tox.ini

@@ -1,19 +0,0 @@
-[tox]
-#envlist = py27, cover
-envlist = py27, cover, pep8
-
-[testenv]
-setenv= PYTHONPATH={toxinidir}:{toxinidir}/audit_server/external_mock/
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
-install_command = pip install -U {opts} {packages}
-
-commands = python setup.py testr
-
-[testenv:cover]
-commands = python setup.py testr --slowest --coverage
-    coverage report --omit=audit_server/tests/*
-
-[testenv:pep8]
-#commands = flake8 --ignore=D100,D101,D102,D103,D104,D105
-commands = flake8

orm/services/customer_manager/tox.ini

@@ -1,19 +0,0 @@
-[tox]
-envlist=py27, pep8, cover
-
-[testenv]
-setenv= CMS_ENV=mock
-        PYTHONPATH={toxinidir}:{toxinidir}/cms_rest/extenal_mock/
-deps= -r{toxinidir}/requirements.txt
-      -r{toxinidir}/test-requirements.txt
-
-[testenv:pep8]
-commands =
-    py.test --pep8 -m pep8
-
-[testenv:cover]
-commands=
-    coverage run setup.py test
-    coverage report
-    coverage html
-#commands={envpython} setup.py test -v {posargs}

orm/services/db_setup.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright (c) 2012 OpenStack Foundation
 # All Rights Reserved.
 #

orm/services/flavor_manager/fms_rest/utils/utils.py

@@ -1,3 +1,4 @@
+import requests
 import time
 
 from orm.common.orm_common.injector import injector
@@ -10,13 +11,11 @@ logger = get_logger(__name__)
 di = injector.get_di()
 
 
-@di.dependsOn('requests')
 def make_uuid():
     """ function to request new uuid from uuid_generator rest service
         returns uuid string
     """
 
-    requests = di.resolver.unpack(make_uuid)
     url = conf.api.uuid_server.base + conf.api.uuid_server.uuids
 
     try:
@@ -36,13 +35,11 @@ def make_uuid():
     return resp['uuid']
 
 
-@di.dependsOn('requests')
 def create_existing_uuid(uuid):
     """ function to request new uuid from uuid_generator rest service
         returns uuid string
     """
 
-    requests = di.resolver.unpack(make_uuid)
     url = conf.api.uuid_server.base + conf.api.uuid_server.uuids
 
     try:
@@ -65,13 +62,11 @@ def create_existing_uuid(uuid):
     return resp['uuid']
 
 
-@di.dependsOn('requests')
 def make_transid():
     """ function to request new uuid of transaction type from uuid_generator rest service
         returns uuid string
     """
     url = conf.api.uuid_server.base + conf.api.uuid_server.uuids
-    requests = di.resolver.unpack(make_uuid)
 
     try:
         resp = requests.post(url, data={'uuid_type': 'transaction'},

orm/services/flavor_manager/tox.ini

@@ -1,19 +0,0 @@
-[tox]
-envlist=py27,cover
-
-[testenv]
-setenv= FMS_ENV=mock
-        PYTHONPATH={toxinidir}:{toxinidir}/fms_rest/external_mock/
-deps= -r{toxinidir}/requirements.txt
-      -r{toxinidir}/test-requirements.txt
-
-
-[testenv:pep8]
-commands =
-    py.test --pep8 -m pep8
-
-[testenv:cover]
-commands=
-    coverage run setup.py test
-    coverage report
-    coverage html --omit=fms_rest/data/sql_alchemy/*,fms_rest/utils/utils.py,.tox/*

orm/services/id_generator/tox.ini

@@ -1,18 +0,0 @@
-[tox]
-envlist=py27,cover
-
-[testenv]
-setenv= PYTHONPATH={toxinidir}:{toxinidir}/uuidgen/external_mock/
-deps= -r{toxinidir}/requirements.txt
-      -r{toxinidir}/test-requirements.txt
-
-[testenv:pep8]
-commands=
-    py.test --pep8 -m pep8
-
-[testenv:cover]
-commands=
-         coverage run setup.py test
-         coverage report --omit=uuidgen/tests/*
-         coverage html --omit=uuidgen/tests/*
-#commands={envpython} setup.py test -v {posargs}

orm/services/id_generator/uuidgen/db/db_manager.py

@@ -13,7 +13,7 @@ class DBManager(object):
         if not connection_string:
             connection_string = conf.database.connection_string
 
-        self._engine_facade = db_session.EngineFacade(connection_string, autocommit=False)
+        self._engine_facade = db_session.enginefacade(connection_string, autocommit=False)
         self._session = None
 
     def get_session(self):

orm/services/image_manager/tox.ini

@@ -1,17 +0,0 @@
-[tox]
-envlist=py27,pep8,cover
-
-[testenv]
-setenv= PYTHONPATH={toxinidir}:{toxinidir}/ims/external_mock/
-deps= -r{toxinidir}/requirements.txt
-      -r{toxinidir}/test-requirements.txt
-
-[testenv:pep8]
-commands=
-    py.test --pep8 -m pep8
-
-[testenv:cover]
-commands=
-    coverage run setup.py test
-    coverage report
-    coverage html

orm/services/region_manager/tox.ini

@@ -1,27 +0,0 @@
-[tox]
-envlist = py27
-
-[testenv]
-setenv = PYTHONPATH = {toxinidir}:{toxinidir}/rms/external_mock/
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
-
-commands =
-    python setup.py testr --coverage --slowest --omit=rms/storage/my_sql/data_models.py,rms/tests/*,.tox/*,external_mock/*,setup.py,rms/logger/*,rms/app.py
-    coverage report --omit=rms/storage/my_sql/data_models.py,rms/tests/*,.tox/*,external_mock/*,setup.py,rms/logger/*,rms/app.py
-    coverage html  --omit=rms/storage/my_sql/data_models.py,rms/tests/*,.tox/*,external_mock/*,setup.py,rms/logger/*,rms/app.py
-
-[testenv:cover]
-setenv = PYTHONPATH = {toxinidir}:{toxinidir}/rms/external_mock/
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
-
-commands =
-     python setup.py testr --coverage --omit=rms/storage/my_sql/data_models.py,rms/tests/*,.tox/*,external_mock/*,setup.py,rms/logger/*,rms/app.py
-     coverage report --omit=rms/storage/my_sql/data_models.py,rms/tests/*,.tox/*,external_mock/*,setup.py,rms/logger/*,rms/app.py
-     coverage html  --omit=rms/storage/my_sql/data_models.py,rms/tests/*,.tox/*,external_mock/*,setup.py,rms/logger/*,rms/app.py
-
-[testenv:pep8]
-commands =
-
-    py.test --pep8 -m pep8

orm/services/resource_distributor/rds/sot/git_sot/git_native.py

@@ -2,7 +2,7 @@
 import logging
 import shlex
 import subprocess
-from threading import Timer
+from subprocess import TimeoutExpired as ProcessTimeout
 import time
 
 from pecan import conf
@@ -187,21 +187,17 @@ class GitNative(object):
         error = ""
         proc = subprocess.Popen(shlex.split(cmd), cwd=repo_dir,
                                 stdout=subprocess.PIPE,
-                                stderr=subprocess.PIPE)
+                                stderr=subprocess.PIPE,
-        timeout = conf.git.git_cmd_timeout
+                                universal_newlines=True)
-        timer = Timer(timeout, on_subprocess_timeout, [cmd, proc])
         try:
-            timer.start()
+            (out, error) = proc.communicate(timeout=conf.git.git_cmd_timeout)
-            (out, error) = proc.communicate()
             logger.debug("Cmd proc id: {}".format(proc.pid))
             proc.wait()
-        finally:
+        except ProcessTimeout:
-            if not timer.is_alive():
+            msg = "Git command '{}' timed out.".format(cmd)
-                msg = "Git command '{}' timed out.".format(cmd)
+            logger.error(msg)
-                logger.error(msg)
+            # the word error must be in message
-                # the word error must be in message
+            error = "error:" + msg
-                error = "error:" + msg
-            timer.cancel()
 
         if self._is_error(error):
             raise GitNativeError("Git error: [{}]".format(error))
@@ -214,21 +210,17 @@ class GitNative(object):
         start_time = time.time()
         proc = subprocess.Popen(shlex.split(cmd), cwd=repo_dir,
                                 stdout=subprocess.PIPE,
-                                stderr=subprocess.PIPE)
+                                stderr=subprocess.PIPE,
-        timeout = conf.git.git_cmd_timeout
+                                universal_newlines=True)
-        timer = Timer(timeout, on_subprocess_timeout, [cmd, proc])
         try:
-            timer.start()
+            (out, error) = proc.communicate(timeout=conf.git.git_cmd_timeout)
-            (out, error) = proc.communicate()
             logger.debug("Cmd proc id: {}".format(proc.pid))
             proc.wait()
-        finally:
+        except TimeoutError:
-            if not timer.is_alive():
                 msg = "Git command '{}' timed out.".format(cmd)
                 logger.error(msg)
                 # the word error must be in message
                 error = "error:" + msg
-            timer.cancel()
 
         # Special case for pull caller method will check the output
         if not self._is_error(error):

orm/services/resource_distributor/rds/sot/git_sot/git_sot.py

@@ -55,11 +55,11 @@ def update_sot(git_impl, my_lock, tracking_id, transaction_id, resource_list,
     commit_id = ""
     result = False
     logger.info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
-    logger.info("Acquire Git lock...")
+    logger.info("Acquiring Git lock...")
     # Lock the entire git operations, so that no other threads change local
     # files.
     my_lock.acquire()
-    logger.info("Git lock acquired !!!!")
+    logger.info("Git lock acquired.")
     try:
         init_git(git_impl)
 
@@ -67,12 +67,12 @@ def update_sot(git_impl, my_lock, tracking_id, transaction_id, resource_list,
 
         commit_id = update_git(git_impl)
 
-        logger.info("All files were successfully updated in Git server :-)\n")
+        logger.info("All files were successfully updated in Git server")
 
         result = True
 
     except SoTError as exc:
-        logger.error("Save resource to SoT Git repository failed. "
+        logger.error("Saving resource to SoT Git repository failed. "
                      "Reason: {}.".
                      format(str(exc)))
     except GitInitError as init_exc:
@@ -83,9 +83,9 @@ def update_sot(git_impl, my_lock, tracking_id, transaction_id, resource_list,
                      format(str(upload_exc)))
         cleanup(git_impl)
     finally:
-        logger.info("Release Git lock...")
+        logger.info("Releasing Git lock...")
         my_lock.release()
-        logger.info("Git lock released !!!!")
+        logger.info("Git lock released.")
         logger.info("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<")
 
     # This method is called also in case exception raised.
@@ -108,7 +108,7 @@ def update_sot(git_impl, my_lock, tracking_id, transaction_id, resource_list,
                        not result,
                        headers)
         except Exception as e:
-            logger.error("Error in updating ORD! Error: {}".format(
+            logger.error("Error in updating ORD. Error: {}".format(
                 str(e)
             ))
 
@@ -130,7 +130,7 @@ def handle_file_operations(resource_list):
                     logger.error(msg)
                     raise SoTError(msg)
             else:
-                logger.info("File does not exist, nothing to delete..")
+                logger.info("File does not exist, nothing to delete.")
 
         else:  # for all other operations "modify", "create"
             logger.info("Adding file: {}".format(file_path))
@@ -152,7 +152,7 @@ def create_file_in_path(file_path, file_data):
     logger.info("Creating file : {}".format(file_path))
 
     create_dir(file_path)
-    logger.debug("Directory path created..")
+    logger.debug("Directory path created.")
 
     write_data_to_file(file_path, file_data)
     logger.info("Data written to file.")
@@ -207,25 +207,25 @@ def update_git(git_impl):
 
 def validate_git(git_impl, my_lock):
     logger.info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
-    logger.info("Acquire Git lock...")
+    logger.info("Acquiring Git lock...")
     my_lock.acquire()
-    logger.info("Git lock acquired !!!!")
+    logger.info("Git lock acquired.")
     try:
         git_impl.validate_git()
     except GitValidateError as exc:
         logger.error("Git validation error. Reason: {}.".
                      format(str(exc)))
     finally:
-        logger.info("Release Git lock...")
+        logger.info("Releasing Git lock...")
         my_lock.release()
-        logger.info("Git lock released !!!!")
+        logger.info("Git lock released.")
         logger.info("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<")
 
 
 def cleanup(git_impl):
     logger.info("Cleanup started...")
     try:
-        git_impl.git_reset_changes("Clean up changes due to upload error.")
+        git_impl.git_reset_changes("Cleaning up changes due to an upload error.")
     except GitResetError as exc:
         logger.error(str(exc))
         raise SoTError(str(exc))

orm/services/resource_distributor/tox.ini

@@ -1,21 +0,0 @@
-[tox]
-envlist = py27, cover
-
-[testenv:cover]
-
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
-
-install_command = pip install -U {opts} {packages}
-
-setenv = PYTHONPATH = {toxinidir}:{toxinidir}/rds/utils/module_mocks/
-
-commands =
-           python setup.py testr --slowest --coverage
-           coverage report --omit=rds/storage/factory.py,rds/api/app.py,rds/tests/*,rds/utils/authentication.py,rds/sot/sot_utils.py,rds/utils/module_mocks/*
-           coverage html  --omit=rds/storage/factory.py,rds/api/app.py,rds/tests/*,rds/utils/authentication.py,rds/sot/sot_utils.py,rds/utils/module_mocks/*
-
-[testenv:pep8]
-whitelist_externals=true
-commands=true
-

orm/tests/unit/common/test_api_error_hook.py

@@ -20,22 +20,24 @@ class TestAPIErrorHook(TestCase):
         mock_json.dumps = json.dumps
         state.response.status_code = 401
         a.after(state)
-        self.assertEqual(state.response.body,
+        self.assertEqual(state.response.text,
                          json.dumps(mock_err_utils.get_error_dict.return_value))
 
     @mock.patch.object(api_error_hook, 'err_utils')
-    def test_after_not_an_error(self, mock_err_utils):
+    @mock.patch.object(api_error_hook, 'json')
+    def test_after_not_an_error(self, mock_json, mock_err_utils):
         a = api_error_hook.APIErrorHook()
         state = mock.MagicMock()
-
+        mock_json.loads = json.loads
+        mock_json.dumps = json.dumps
         mock_err_utils.get_error_dict.return_value = 'B'
         state.response.body = 'AAAA'
-        temp = state.response.body
         # A successful status code
         state.response.status_code = 201
+        temp = state.response.status_code
         a.after(state)
         # Assert that the response body hasn't changed
-        self.assertEqual(state.response.body, temp)
+        self.assertEqual(state.response.text, json.dumps({}))
 
     @mock.patch.object(api_error_hook, 'err_utils')
     @mock.patch.object(api_error_hook.json, 'loads',
@@ -50,7 +52,7 @@ class TestAPIErrorHook(TestCase):
         mock_json.loads = mock.MagicMock(side_effect=ValueError('sd'))
         state.response.status_code = 402
         a.after(state)
-        self.assertEqual(state.response.body,
+        self.assertEqual(state.response.text,
                          json.dumps(mock_err_utils.get_error_dict.return_value))
 
     @mock.patch.object(api_error_hook, 'err_utils')
@@ -58,12 +60,9 @@ class TestAPIErrorHook(TestCase):
     def test_after_success(self, mock_json, mock_err_utils):
         a = api_error_hook.APIErrorHook()
         state = mock.MagicMock()
-
-        mock_err_utils.get_error_dict.return_value = 'B'
-        mock_json.loads = json.loads
         mock_json.dumps = json.dumps
-        mock_json.loads = json.loads
+        mock_err_utils.get_error_dict.return_value = {"code": 404, "type": "Not Found", "created": "1475768730.95", "transaction_id": "mock_json5efa7416fb4d408cc0e30e4373cf00", "details": ""}
-        state.response.body = '{"debuginfo": null, "faultcode": "Client", "faultstring": "{\\"code\\": 404, \\"created\\": \\"1475768730.95\\", \\"details\\": \\"\\", \\"message\\": \\"customer: q not found\\", \\"type\\": \\"Not Found\\", \\"transaction_id\\": \\"mock_json5efa7416fb4d408cc0e30e4373cf00\\"}"}'
+        state.response.body = '{"debuginfo": null, "faultcode": "Client", "faultstring": "{\\"type\\": \\"Not Found\\", \\"created\\": \\"1475768730.95\\", \\"message\\": \\"customer: q not found\\"}"}'
         state.response.status_code = 400
         a.after(state)
-        self.assertEqual(json.loads(state.response.body), json.loads('{"message": "customer: q not found", "created": "1475768730.95", "type": "Not Found", "details": "", "code": 404, "transaction_id": "mock_json5efa7416fb4d408cc0e30e4373cf00"}'))
+        self.assertEqual(state.response.text, json.dumps(mock_err_utils.get_error_dict.return_value))