x/ranger-agent
Code Issues Proposed changes

Merge Python3 branch

Browse Source 
This change is going to upgrade ranger-agent to use Python 3.x

Change-Id: Ib7453eb32838672254d85f4981cab33dd816f75c
This commit is contained in:
hosingh000 2019-10-28 15:56:54 -05:00
parent 02114b616b
commit b0c32e7e47
30 changed files with 110 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.zuul.yaml
View File

@ -2,8 +2,9 @@
check:
jobs:
- openstack-tox-pep8
- openstack-tox-py27
- openstack-tox-py36
gate:
jobs:
- openstack-tox-pep8
- openstack-tox-py27
- openstack-tox-py36

20
Dockerfile
View File

@ -1,4 +1,4 @@
FROM ubuntu:16.04
FROM ubuntu:18.04
ENV DEBIAN_FRONTEND noninteractive
ENV container docker
@ -11,11 +11,11 @@ apt -y install git \
netcat \
netbase \
openssh-server \
python-minimal \
python-setuptools \
python-pip \
python-dev \
python-dateutil \
python3-minimal \
python3-setuptools \
python3-pip \
python3-dev \
python3-dateutil \
ca-certificates \
openstack-pkg-tools \
gcc \
@ -31,16 +31,16 @@ libssl-dev --no-install-recommends \
/usr/share/doc \
/usr/share/doc-base
RUN pip install --upgrade pip
RUN pip install wheel
RUN pip3 install --upgrade pip
RUN pip3 install wheel
COPY . /tmp/ranger-agent
WORKDIR /tmp/ranger-agent
RUN pip install --default-timeout=100 -r requirements.txt
RUN pip3 install --default-timeout=100 -r requirements.txt
RUN python setup.py install
RUN python3 setup.py install
ARG user=ranger_agent

18
doc/source/conf.py
View File

@ -67,15 +67,15 @@ source_suffix = '.rst'
master_doc = 'index'
# General information about the project.
project = u'ranger-agent'
copyright = u'2011-present, OpenStack, LLC.'
project = 'ranger-agent'
copyright = '2011-present, OpenStack, LLC.'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
from ord import __version__
from ord import __version__
# The full version, including alpha/beta/rc tags.
release = __version__
# The short X.Y version.
@ -124,10 +124,10 @@ modindex_common_prefix = ['ranger-agent.']
# List of tuples 'sourcefile', 'target', u'title', u'Authors name', 'manual'
man_pages = [
('man/rangeragentapi', 'ranger-agent-api', u'Ranger Agent API Server',
[u'OpenStack'], 1),
('man/rangeragentengine', 'ranger-agent-registry', u'Ranger Agent Engine Server',
[u'OpenStack'], 1)
('man/rangeragentapi', 'ranger-agent-api', 'Ranger Agent API Server',
['OpenStack'], 1),
('man/rangeragentengine', 'ranger-agent-registry', 'Ranger Agent Engine Server',
['OpenStack'], 1)
]
@ -218,8 +218,8 @@ htmlhelp_basename = '%sdoc' % project
# (source start file, target name, title, author,
# documentclass [howto/manual]).
latex_documents = [
('index', 'ranger-agent.tex', u'Ranger agent Documentation',
u'Ranger Team', 'manual'),
('index', 'ranger-agent.tex', 'Ranger agent Documentation',
'Ranger Team', 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of

19
ord/api/controllers/v1/api.py
View File

@ -23,12 +23,14 @@ from ord.i18n import _
from ord.openstack.common import log
from oslo_config import cfg
from pecan import expose
from urllib2 import HTTPError
from urllib.error import HTTPError
import datetime
import json
import oslo_messaging as messaging
import urllib2
import urllib
from urllib import request
import uuid
import webob
import webob.exc
@ -70,10 +72,13 @@ class ListenerQueueHandler(object):
LOG.debug(" Payload to RDS Listener %s " % listener_response_body)
headers = {'Content-type': 'application/json'}
rds_url = CONF.orm.rds_listener_endpoint
req = urllib2.Request(rds_url, # nosec
json.dumps(listener_response_body),
headers,
unverifiable=False)
# Python3 urllib: convert listener_response_body to bytes
response_body_bytes = \
json.dumps(listener_response_body).encode("utf-8")
req = urllib.request.Request(rds_url, # nosec
response_body_bytes,
headers,
unverifiable=False)
args = {}
template_status_id = None
if 'rds-listener' in listener_response_body:
@ -88,7 +93,7 @@ class ListenerQueueHandler(object):
status_code = None
try:
LOG.info('Connecting to RDS at %s' % rds_url)
resp = urllib2.urlopen(req) # nosec
resp = request.urlopen(req) # nosec
status = utils.STATUS_RDS_SUCCESS
if resp is not None:
status_code = resp.getcode()

1
ord/client/getrepo.py
View File

@ -136,7 +136,6 @@ class TemplateRepoClient(object):
process = subprocess.Popen(
shlex.split(cmd), stdout=subprocess.PIPE,
shell=False, stderr=subprocess.PIPE)
[stdout, stderr] = process.communicate()
# 124 is the return code in the shell if timeout occurred

2
ord/cmd/__init__.py
View File

@ -15,4 +15,4 @@
import eventlet
eventlet.monkey_patch(socket=True, select=True, thread=True, time=True)
eventlet.monkey_patch()

4
ord/cmd/fake_rds_listener.py
View File

@ -89,7 +89,7 @@ class _CatcherStorage(object):
break
if till:
for idx in xrange(len(self.data) - 1, sidx - 1, -1):
for idx in range(len(self.data) - 1, sidx - 1, -1):
entity = self.data[idx]
if entity['idnr'] != till:
continue
@ -144,7 +144,7 @@ class _NotifierCatcher(_HandlerBase):
if len(payload) != 1:
return payload
key = payload.keys()[0]
key = list(payload.keys())[0]
value = payload[key]
if value != ['']:
return payload

2
ord/cmd/manage.py
View File

@ -34,4 +34,4 @@ def dbsync():
def version():
service.prepare_service()
"""Print the current database version."""
print(mig.db_version(database='ord'))
print((mig.db_version(database='ord')))

2
ord/common/security_headers_hook.py
View File

@ -15,5 +15,5 @@ class SecurityHeadersHook(PecanHook):
'X-XSS-Protection': '1; mode=block'}
# Add all the security headers
for header, value in security_headers.items():
for header, value in list(security_headers.items()):
state.response.headers.add(header, value)

5
ord/common/utils.py
View File

@ -151,11 +151,6 @@ class ErrorCode(Enum):
ORD_018 = "ORD_018"
ORD_019 = "ORD_019"
def __getattr__(self, code):
if code in self:
return code
raise AttributeError
@classmethod
def tostring(cls, errorCode):
ord_err = {'ORD_000': 'Stack Creation Failed',

2
ord/db/model_base.py
View File

@ -28,7 +28,7 @@ class ORDBase(models.ModelBase):
self._i = iter(orm.object_mapper(self).columns)
return self
def next(self):
def __next__(self):
n = self._i.next().name
return n, getattr(self, n)

4
ord/engine/healthcheck.py
View File

@ -59,9 +59,9 @@ class HealthCheck(object):
LOG.debug("Health Heat test starting")
heat.HeatClient().delete_stack('none_existence_id')
except exc.HEATStackDeleteError as heatex:
if 'MessagingTimeout' in heatex.message:
if 'MessagingTimeout' in str(heatex):
LOG.error('Health Heat Test Exp in %s: %r',
cls.__name__, heatex, exc_info=True)
cls.__name__, str(heatex), exc_info=True)
status['heat'] = 'failed'
else:
status['heat'] = 'passed'

3
ord/engine/workerfactory.py
View File

@ -12,7 +12,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import itertools
import json
import os
from oslo_config import cfg
@ -465,7 +464,7 @@ class StatusTransitions(object):
def __str__(self):
chunks = []
for status, hits in itertools.izip(self.transitions, self.hits):
for status, hits in zip(self.transitions, self.hits):
if 1 < hits:
status = '{}({})'.format(status, hits)
chunks.append(status)

2
ord/hacking/checks.py
View File

@ -49,7 +49,7 @@ def _regex_for_level(level, hint):
log_translation_hint = re.compile(
'|'.join('(?:%s)' % _regex_for_level(level, hint)
for level, hint in _all_log_levels.iteritems()))
for level, hint in _all_log_levels.items()))
oslo_namespace_imports_dot = re.compile(r"import[\s]+oslo[.][^\s]+")
oslo_namespace_imports_from_dot = re.compile(r"from[\s]+oslo[.]")

4
ord/openstack/common/log.py
View File

@ -446,7 +446,7 @@ def _find_facility_from_conf():
facility = facility_names.get(CONF.syslog_log_facility)
if facility is None:
valid_facilities = facility_names.keys()
valid_facilities = list(facility_names.keys())
consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON',
'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS',
'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP',
@ -638,7 +638,7 @@ class ContextFormatter(logging.Formatter):
context = getattr(local.store, 'context', None)
if context:
d = _dictify_context(context)
for k, v in d.items():
for k, v in list(d.items()):
setattr(record, k, v)
# NOTE(sdague): default the fancier formatting params

2
ord/tests/base.py
View File

@ -51,7 +51,7 @@ class BaseTestCase(testtools.TestCase):
fd = cls(prefix=prefix)
if payload:
fd.write(payload)
fd.write(payload.encode('utf-8'))
fd.seek(0, os.SEEK_SET)
return fd

30
ord/tests/unit/api/controllers/v1/test_api.py
View File

@ -17,13 +17,13 @@
Unit Tests for ord.api.test_api
"""
import mock
from mox import stubout
from mox3.mox import stubout
from ord.api.controllers.v1 import api
from ord.db import api as db_api
from ord.tests import base
from oslo_config import cfg
import requests
import urllib2
from urllib import request
import webob
CONF = cfg.CONF
@ -60,14 +60,14 @@ class OrdApiTestCase(base.BaseTestCase):
"region": "local"}
}
db_response = {'template_type': u'hot',
db_response = {'template_type': 'hot',
'status': 'Submitted',
'resource_name': u'image1',
'resource_operation': u'create',
'resource_template_version': u'1',
'request_id': '2', 'region': u'local',
'resource_id': u'1',
'resource_type': u'image',
'resource_name': 'image1',
'resource_operation': 'create',
'resource_template_version': '1',
'request_id': '2', 'region': 'local',
'resource_id': '1',
'resource_type': 'image',
'template_status_id': '1234'}
CONF.set_default('region', 'local')
@ -107,10 +107,10 @@ class OrdApiTestCase(base.BaseTestCase):
def fake_update_target(*args, **kwds):
return db_template_target
self.stubs.Set(urllib2, 'urlopen', mock_url_open)
self.stubs.Set(request, 'urlopen', mock_url_open)
self.stubs.Set(db_api, "update_target_data",
fake_update_target)
self.stubs.Set(urllib2, 'Request', urlrequest_mock_method)
self.stubs.Set(request, 'Request', urlrequest_mock_method)
api_listener.invoke_listener_rpc(ctxt, payload)
def test_rds_listener_failure(self):
@ -131,7 +131,7 @@ class OrdApiTestCase(base.BaseTestCase):
def mock_method(url, payload, headers, unverifiable=False):
return "Failure"
self.stubs.Set(urllib2, 'Request', mock_method)
self.stubs.Set(request, 'Request', mock_method)
def mock_url_open(mock_response):
mock_response = mock.Mock()
@ -141,7 +141,7 @@ class OrdApiTestCase(base.BaseTestCase):
def fake_update_target(*args, **kwds):
return db_template_target
self.stubs.Set(urllib2, 'urlopen', mock_url_open)
self.stubs.Set(request, 'urlopen', mock_url_open)
self.stubs.Set(db_api, "update_target_data",
fake_update_target)
api_listener.invoke_listener_rpc(ctxt, payload)
@ -165,7 +165,7 @@ class OrdApiTestCase(base.BaseTestCase):
def mock_method(url, payload, headers, unverifiable=False):
return "Success"
self.stubs.Set(urllib2, 'Request', mock_method)
self.stubs.Set(request, 'Request', mock_method)
def mock_url_open(mock_response):
mock_response = mock.Mock()
@ -174,7 +174,7 @@ class OrdApiTestCase(base.BaseTestCase):
def fake_update_target(*args, **kwds):
return db_template_target
self.stubs.Set(urllib2, 'urlopen', mock_url_open)
self.stubs.Set(request, 'urlopen', mock_url_open)
self.stubs.Set(db_api, "update_target_data",
fake_update_target)
api_listener.invoke_listener_rpc(ctxt, payload)

10
ord/tests/unit/client/test_getrepo.py
View File

@ -13,15 +13,19 @@
# License for the specific language governing permissions and limitations
# under the License.
import mock
from mock import patch
import os
from ord.client import getrepo
from ord.common.exceptions import ORDException
from ord.tests import base
import os
from oslo_config import cfg
from unittest import mock
from unittest.mock import patch
CONF = cfg.CONF

4
ord/tests/unit/client/test_heat.py
View File

@ -14,11 +14,11 @@
# under the License.
from heatclient import exc as heat_exc
import mock
from ord.client import heat as ord_heat
from ord.common import exceptions as exc
from ord.tests import base
from ord.common import exceptions as exc
from unittest import mock
class TestHeatClient(base.BaseTestCase):

2
ord/tests/unit/client/test_rpcapi.py
View File

@ -17,10 +17,10 @@
Unit Tests for ord.client.rpcapi
"""
import copy
from mox3 import stubout
from ord.client import rpcapi
from ord.tests import base
from oslo_config import cfg
import stubout
CONF = cfg.CONF

2
ord/tests/unit/client/test_rpcengine.py
View File

@ -17,10 +17,10 @@
Unit Tests for ord.client.rpcengine
"""
import copy
from mox3 import stubout
from ord.client import rpcengine
from ord.tests import base
from oslo_config import cfg
import stubout
CONF = cfg.CONF

2
ord/tests/unit/common/test_utils.py
View File

@ -33,7 +33,7 @@ class TestUtils(base.BaseTestCase):
def test_load_file_fail(self):
temp = self.make_tempfile(payload='payload')
error = IOError(errno.ENOENT, os.strerror(errno.ENOENT), temp.name)
self.patch('__builtin__.open', side_effect=error)
self.patch('builtins.open', side_effect=error)
self.assertRaises(exc.InternalError, utils.load_file, temp.name)

4
releasenotes/source/conf.py
View File

@ -24,9 +24,9 @@ copyright = '2018, Ranger Authors'
author = 'Ranger Authors'
# The short X.Y version
version = u'0.1.0'
version = '0.1.0'
# The full version, including alpha/beta/rc tags
release = u'0.1.0'
release = '0.1.0'
# -- General configuration ---------------------------------------------------

25
requirements.txt
View File

@ -3,23 +3,22 @@
# process, which may cause wedges in the gate later.
#
pbr>=1.8
oslo.config>=3.14.0
oslo.config>=5.2.0 # Apache-2.0
oslo.messaging>=5.29.0
oslo.serialization>=1.10.0
oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0
oslo.db>=4.15.0
oslo.log>=3.11.0
oslo.utils>=3.18.0
oslo.log>=3.36.0 # Apache-2.0
oslo.utils>=3.37.0 # Apache-2.0
pecan>=1.0.0,!=1.0.2,!=1.0.3,!=1.0.4
python-heatclient>=1.6.1 # Apache-2.0
python-glanceclient>=2.5.0 # Apache-2.0
python-heatclient>=1.10.0 # Apache-2.0
python-glanceclient>=2.8.0 # Apache-2.0
python-keystoneclient>=3.8.0 # Apache-2.0
werkzeug>=0.11.5
eventlet!=0.18.3,>=0.18.2
SQLAlchemy==1.1.0
sqlalchemy-migrate>=0.9.6
PyMySQL>=0.7.6
lxml>=2.3,!=3.7.0
PasteDeploy>=1.5.0
eventlet>=0.24.1
SQLAlchemy<1.3.0,>=1.2.0
sqlalchemy-migrate>=0.11.0 # Apache-2.0
PyMySQL==0.9.3
lxml==4.3.2
PasteDeploy==2.0.1
psutil>=3.0.1
enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3'

5
setup.cfg
View File

@ -14,9 +14,8 @@ classifier =
License :: OSI Approved :: Apache Software License
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.6
[global]
setup-hooks =

2
test-requirements.txt
View File

@ -14,6 +14,6 @@ testscenarios>=0.4
testtools>=1.4.0
mock>=2.0
discover
mox>=0.5.3
mox3>=0.27.0
psycopg2>=2.5
reno>=1.8.0 # Apache-2.0

21
tools/colorizer.py
View File

@ -168,9 +168,9 @@ def get_elapsed_time_color(elapsed_time):
return 'green'
class NovaTestResult(testtools.TestResult):
class RangerAgentTestResult(testtools.TestResult):
def __init__(self, stream, descriptions, verbosity):
super(NovaTestResult, self).__init__()
super(RangerAgentTestResult, self).__init__()
self.stream = stream
self.showAll = verbosity > 1
self.num_slow_tests = 10
@ -224,26 +224,26 @@ class NovaTestResult(testtools.TestResult):
self.colorizer.write(short_result, color)
def addSuccess(self, test):
super(NovaTestResult, self).addSuccess(test)
super(RangerAgentTestResult, self).addSuccess(test)
self._addResult(test, 'OK', 'green', '.', True)
def addFailure(self, test, err):
if test.id() == 'process-returncode':
return
super(NovaTestResult, self).addFailure(test, err)
super(RangerAgentTestResult, self).addFailure(test, err)
self._addResult(test, 'FAIL', 'red', 'F', False)
def addError(self, test, err):
super(NovaTestResult, self).addFailure(test, err)
super(RangerAgentTestResult, self).addFailure(test, err)
self._addResult(test, 'ERROR', 'red', 'E', False)
def addSkip(self, test, reason=None, details=None):
super(NovaTestResult, self).addSkip(test, reason, details)
super(RangerAgentTestResult, self).addSkip(test, reason, details)
self._addResult(test, 'SKIP', 'blue', 'S', True)
def startTest(self, test):
self.start_time = self._now()
super(NovaTestResult, self).startTest(test)
super(RangerAgentTestResult, self).startTest(test)
def writeTestCase(self, cls):
if not self.results.get(cls):
@ -262,7 +262,7 @@ class NovaTestResult(testtools.TestResult):
if not self.last_written or (self._now() - time).total_seconds() > 2.0:
diff = 3.0
while diff > 2.0:
classes = self.results.keys()
classes = list(self.results.keys())
oldest = min(classes, key=lambda x: self.last_time[x])
diff = (self._now() - self.last_time[oldest]).total_seconds()
self.writeTestCase(oldest)
@ -273,7 +273,7 @@ class NovaTestResult(testtools.TestResult):
self.stopTestRun()
def stopTestRun(self):
for cls in list(self.results.iterkeys()):
for cls in list(self.results.keys()):
self.writeTestCase(cls)
self.stream.writeln()
self.writeSlowTests()
@ -322,7 +322,8 @@ test = subunit.ProtocolTestCase(sys.stdin, passthrough=None)
if sys.version_info[0:2] <= (2, 6):
runner = unittest.TextTestRunner(verbosity=2)
else:
runner = unittest.TextTestRunner(verbosity=2, resultclass=NovaTestResult)
runner = unittest.TextTestRunner(verbosity=2,
resultclass=RangerAgentTestResult)
if runner.run(test).wasSuccessful():
exit_code = 0

14
tools/install_venv.py
View File

@ -17,8 +17,6 @@
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import print_function
import os
import sys
@ -27,13 +25,13 @@ import install_venv_common as install_venv
def print_help(venv, root):
help = """
Nova development environment setup is complete.
Ranger Agent development environment setup is complete.
Nova development uses virtualenv to track and manage Python dependencies
while in development and testing.
Ranger Agent development uses virtualenv to track and manage Python
dependencies while in development and testing.
To activate the Nova virtualenv for the extent of your current shell
session you can run:
To activate the ranger-agent virtualenv for the extent of your current
shell session you can run:
$ source %s/bin/activate
@ -59,7 +57,7 @@ def main(argv):
pip_requires = os.path.join(root, 'requirements.txt')
test_requires = os.path.join(root, 'test-requirements.txt')
py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1])
project = 'Nova'
project = 'ranger-agent'
install = install_venv.InstallVenv(root, venv, pip_requires, test_requires,
py_version, project)
options = install.parse_args(argv)

2
tools/install_venv_common.py
View File

@ -22,8 +22,6 @@ environment, it should be kept strictly compatible with Python 2.6.
Synced in from openstack-common
"""
from __future__ import print_function
import optparse
import os
import subprocess

3
tox.ini
View File

@ -1,10 +1,11 @@
[tox]
minversion = 2.0
#Removed py35 and pypy not included for now.
envlist = py27,pep8
envlist = py36,pep8
skipsdist = True
[testenv]
basepython = python3.6
usedevelop = True
install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
setenv =