Browse Source

Initial Python 3 support

Packstack needs to get adapted to Python 3. This patch adds initial
compatibility fixes and a tox-py36 job (non-voting) to test it.

Change-Id: I653454b523224615ea5f0e9f5a7d799031f57649
Javier Pena 1 year ago
parent
commit
e2c31de9b3

+ 2
- 0
.zuul.yaml View File

@@ -87,6 +87,8 @@
87 87
         - packstack-integration-scenario002-tempest
88 88
         - packstack-integration-scenario003-tempest
89 89
         - packstack-multinode-scenario002-tempest
90
+        - openstack-tox-py36:
91
+            voting: false
90 92
     gate:
91 93
       jobs:
92 94
         - packstack-integration-scenario001-tempest

+ 1
- 1
docs/packstack.rst View File

@@ -1,4 +1,4 @@
1
-=========
1
+=========
2 2
 Packstack
3 3
 =========
4 4
 

+ 3
- 2
packstack/installer/core/drones.py View File

@@ -13,6 +13,7 @@
13 13
 # limitations under the License.
14 14
 
15 15
 import os
16
+import six
16 17
 import stat
17 18
 import uuid
18 19
 import time
@@ -75,7 +76,7 @@ class SshTarballTransferMixin(object):
75 76
             dest = self.recipe_dir[len(self.resource_dir):].lstrip('/')
76 77
         else:
77 78
             dest = ''
78
-        for marker, recipes in self._recipes.iteritems():
79
+        for marker, recipes in six.iteritems(self._recipes):
79 80
             for path in recipes:
80 81
                 _dest = os.path.join(dest, os.path.basename(path))
81 82
                 pack.add(path, arcname=_dest)
@@ -278,7 +279,7 @@ class Drone(object):
278 279
         logger = logging.getLogger()
279 280
         skip = skip or []
280 281
         lastmarker = None
281
-        for mark, recipelist in self._recipes.iteritems():
282
+        for mark, recipelist in six.iteritems(self._recipes):
282 283
             if marker and marker != mark:
283 284
                 logger.debug('Skipping marker %s for node %s.' %
284 285
                              (mark, self.node))

+ 2
- 1
packstack/installer/core/parameters.py View File

@@ -17,6 +17,7 @@ Container set for groups and parameters
17 17
 """
18 18
 
19 19
 from ..utils.datastructures import SortedDict
20
+import six
20 21
 
21 22
 
22 23
 class Parameter(object):
@@ -31,7 +32,7 @@ class Parameter(object):
31 32
         defaults = {}.fromkeys(self.allowed_keys)
32 33
         defaults.update(attributes)
33 34
 
34
-        for key, value in defaults.iteritems():
35
+        for key, value in six.iteritems(defaults):
35 36
             if key not in self.allowed_keys:
36 37
                 raise KeyError('Given attribute %s is not allowed' % key)
37 38
             self.__dict__[key] = value

+ 1
- 1
packstack/installer/output_messages.py View File

@@ -25,7 +25,7 @@ DONT CHANGE any of the params names (in UPPER-CASE)
25 25
 they are used in the engine-setup.py
26 26
 '''
27 27
 
28
-import basedefs
28
+from packstack.installer import basedefs
29 29
 
30 30
 INFO_HEADER = "Welcome to the %s setup utility" % basedefs.APP_NAME
31 31
 INFO_INSTALL_SUCCESS = "\n **** Installation completed successfully ******\n"

+ 22
- 19
packstack/installer/run_setup.py View File

@@ -11,15 +11,18 @@
11 11
 # See the License for the specific language governing permissions and
12 12
 # limitations under the License.
13 13
 
14
-import ConfigParser
14
+from six.moves import configparser
15
+from six.moves import StringIO
16
+from functools import cmp_to_key
17
+
15 18
 import copy
16 19
 import datetime
17 20
 import getpass
18 21
 import logging
19 22
 import os
20 23
 import re
24
+import six
21 25
 import sys
22
-from StringIO import StringIO
23 26
 import traceback
24 27
 import types
25 28
 import textwrap
@@ -27,17 +30,17 @@ import textwrap
27 30
 from optparse import OptionGroup
28 31
 from optparse import OptionParser
29 32
 
30
-import basedefs
31
-import validators
33
+from packstack.installer import basedefs
34
+from packstack.installer import validators
32 35
 from . import utils
33
-import processors
34
-import output_messages
36
+from packstack.installer import processors
37
+from packstack.installer import output_messages
35 38
 from .exceptions import FlagValidationError
36 39
 from .exceptions import ParamValidationError
37 40
 
38 41
 from packstack.modules.common import filtered_hosts
39 42
 from packstack.version import version_info
40
-from setup_controller import Controller
43
+from packstack.installer.setup_controller import Controller
41 44
 
42 45
 controller = Controller()
43 46
 commandLineValues = {}
@@ -238,21 +241,21 @@ def mask(input):
238 241
     If it finds, it replaces them with '********'
239 242
     """
240 243
     output = copy.deepcopy(input)
241
-    if isinstance(input, types.DictType):
244
+    if isinstance(input, dict):
242 245
         for key in input:
243
-            if isinstance(input[key], types.StringType):
246
+            if isinstance(input[key], six.string_types):
244 247
                 output[key] = utils.mask_string(input[key],
245 248
                                                 masked_value_set)
246
-    if isinstance(input, types.ListType):
249
+    if isinstance(input, list):
247 250
         for item in input:
248 251
             org = item
249 252
             orgIndex = input.index(org)
250
-            if isinstance(item, types.StringType):
253
+            if isinstance(item, six.string_types):
251 254
                 item = utils.mask_string(item, masked_value_set)
252 255
             if item != org:
253 256
                 output.remove(org)
254 257
                 output.insert(orgIndex, item)
255
-    if isinstance(input, types.StringType):
258
+    if isinstance(input, six.string_types):
256 259
             output = utils.mask_string(input, masked_value_set)
257 260
 
258 261
     return output
@@ -329,7 +332,7 @@ def _handleGroupCondition(config, conditionName, conditionValue):
329 332
 
330 333
     # If the condition is a string - just read it to global conf
331 334
     # We assume that if we get a string as a member it is the name of a member of conf_params
332
-    elif isinstance(conditionName, types.StringType):
335
+    elif isinstance(conditionName, six.string_types):
333 336
         conditionValue = _loadParamFromFile(config, "general", conditionName)
334 337
     else:
335 338
         # Any other type is invalid
@@ -349,14 +352,14 @@ def _loadParamFromFile(config, section, param_name):
349 352
     # Get value from answer file
350 353
     try:
351 354
         value = config.get(section, param_name)
352
-    except ConfigParser.NoOptionError:
355
+    except configparser.NoOptionError:
353 356
         value = None
354 357
         # Check for deprecated parameters
355 358
         deprecated = param.DEPRECATES if param.DEPRECATES is not None else []
356 359
         for old_name in deprecated:
357 360
             try:
358 361
                 val = config.get(section, old_name)
359
-            except ConfigParser.NoOptionError:
362
+            except configparser.NoOptionError:
360 363
                 continue
361 364
             if not val:
362 365
                 # value is empty string
@@ -406,7 +409,7 @@ def _handleAnswerFileParams(answerFile):
406 409
         logging.debug("Starting to handle config file")
407 410
 
408 411
         # Read answer file
409
-        fconf = ConfigParser.ConfigParser()
412
+        fconf = configparser.RawConfigParser()
410 413
         fconf.read(answerFile)
411 414
 
412 415
         # Iterate all the groups and check the pre/post conditions
@@ -545,7 +548,7 @@ def _getConditionValue(matchMember):
545 548
     returnValue = False
546 549
     if isinstance(matchMember, types.FunctionType):
547 550
         returnValue = matchMember(controller.CONF)
548
-    elif isinstance(matchMember, types.StringType):
551
+    elif isinstance(matchMember, six.string_types):
549 552
         # we assume that if we get a string as a member it is the name
550 553
         # of a member of conf_params
551 554
         if matchMember not in controller.CONF:
@@ -766,7 +769,7 @@ def validate_answer_file_options(answerfile_path):
766 769
         raise Exception(
767 770
             output_messages.ERR_NO_ANSWER_FILE % answerfile_path)
768 771
 
769
-    answerfile = ConfigParser.ConfigParser()
772
+    answerfile = configparser.RawConfigParser()
770 773
     answerfile.read(answerfile_path)
771 774
     sections = answerfile._sections
772 775
     general_sections = sections.get('general', None)
@@ -912,7 +915,7 @@ def loadPlugins():
912 915
     sys.path.append(basedefs.DIR_MODULES)
913 916
 
914 917
     fileList = [f for f in os.listdir(basedefs.DIR_PLUGINS) if f[0] != "_"]
915
-    fileList = sorted(fileList, cmp=plugin_compare)
918
+    fileList = sorted(fileList, key=cmp_to_key(plugin_compare))
916 919
     for item in fileList:
917 920
         # Looking for files that end with ###.py, example: a_plugin_100.py
918 921
         match = re.search("^(.+\_\d\d\d)\.py$", item)

+ 1
- 1
packstack/installer/utils/datastructures.py View File

@@ -36,7 +36,7 @@ class SortedDict(dict):
36 36
             data = list(data)
37 37
         super(SortedDict, self).__init__(data)
38 38
         if isinstance(data, dict):
39
-            self.keyOrder = data.keys()
39
+            self.keyOrder = list(data)
40 40
         else:
41 41
             self.keyOrder = []
42 42
             seen = set()

+ 1
- 1
packstack/installer/utils/decorators.py View File

@@ -36,6 +36,6 @@ def retry(count=1, delay=0, retry_on=Exception):
36 36
                     if delay:
37 37
                         time.sleep(delay)
38 38
                     tried += 1
39
-        wrapper.func_name = func.func_name
39
+        wrapper.__name__ = func.__name__
40 40
         return wrapper
41 41
     return decorator

+ 16
- 4
packstack/installer/utils/shell.py View File

@@ -14,7 +14,7 @@
14 14
 
15 15
 import re
16 16
 import os
17
-import types
17
+import six
18 18
 import logging
19 19
 import subprocess
20 20
 
@@ -38,11 +38,12 @@ def execute(cmd, workdir=None, can_fail=True, mask_list=None,
38 38
     mask_list = mask_list or []
39 39
     repl_list = [("'", "'\\''")]
40 40
 
41
-    if not isinstance(cmd, types.StringType):
41
+    if not isinstance(cmd, six.string_types):
42 42
         import pipes
43 43
         masked = ' '.join((pipes.quote(i) for i in cmd))
44 44
     else:
45 45
         masked = cmd
46
+
46 47
     masked = mask_string(masked, mask_list, repl_list)
47 48
     if log:
48 49
         logging.info("Executing command:\n%s" % masked)
@@ -53,6 +54,11 @@ def execute(cmd, workdir=None, can_fail=True, mask_list=None,
53 54
                             shell=use_shell, close_fds=True,
54 55
                             env=environ)
55 56
     out, err = proc.communicate()
57
+
58
+    if not isinstance(out, six.text_type):
59
+        out = out.decode('utf-8')
60
+    if not isinstance(err, six.text_type):
61
+        err = err.decode('utf-8')
56 62
     masked_out = mask_string(out, mask_list, repl_list)
57 63
     masked_err = mask_string(err, mask_list, repl_list)
58 64
     if log:
@@ -102,11 +108,17 @@ class ScriptRunner(object):
102 108
             cmd = ["bash", "-x"]
103 109
         environ = os.environ
104 110
         environ['LANG'] = 'en_US.UTF8'
111
+
105 112
         obj = subprocess.Popen(cmd, stdin=_PIPE, stdout=_PIPE, stderr=_PIPE,
106 113
                                close_fds=True, shell=False, env=environ)
107 114
 
108
-        script = "function t(){ exit $? ; } \n trap t ERR \n" + script
109
-        out, err = obj.communicate(script)
115
+        script = "function t(){ exit $? ; } \n trap t ERR \n %s" % script
116
+
117
+        out, err = obj.communicate(script.encode('utf-8'))
118
+        if not isinstance(out, six.text_type):
119
+            out = out.decode('utf-8')
120
+        if not isinstance(err, six.text_type):
121
+            err = err.decode('utf-8')
110 122
         masked_out = mask_string(out, mask_list, repl_list)
111 123
         masked_err = mask_string(err, mask_list, repl_list)
112 124
         if log:

+ 2
- 1
packstack/installer/utils/shortcuts.py View File

@@ -15,10 +15,11 @@
15 15
 import grp
16 16
 import os
17 17
 import pwd
18
+import six
18 19
 
19 20
 
20 21
 def host_iter(config):
21
-    for key, value in config.iteritems():
22
+    for key, value in six.iteritems(config):
22 23
         if key.endswith("_HOST"):
23 24
             host = value.split('/')[0]
24 25
             if host:

+ 16
- 5
packstack/installer/utils/strings.py View File

@@ -12,7 +12,9 @@
12 12
 # See the License for the specific language governing permissions and
13 13
 # limitations under the License.
14 14
 
15
+from functools import cmp_to_key
15 16
 import re
17
+import six
16 18
 
17 19
 
18 20
 STR_MASK = '*' * 8
@@ -29,6 +31,10 @@ def color_text(text, color):
29 31
     return '%s%s%s' % (COLORS[color], text, COLORS['nocolor'])
30 32
 
31 33
 
34
+def stringcmp(x, y):
35
+    return len(y) - len(x)
36
+
37
+
32 38
 def mask_string(unmasked, mask_list=None, replace_list=None):
33 39
     """
34 40
     Replaces words from mask_list with MASK in unmasked string.
@@ -39,14 +45,19 @@ def mask_string(unmasked, mask_list=None, replace_list=None):
39 45
     mask_list = mask_list or []
40 46
     replace_list = replace_list or []
41 47
 
42
-    masked = unmasked
43
-    for word in sorted(mask_list, lambda x, y: len(y) - len(x)):
48
+    if isinstance(unmasked, six.text_type):
49
+        masked = unmasked.encode('utf-8')
50
+    else:
51
+        masked = unmasked
52
+
53
+    for word in sorted(mask_list, key=cmp_to_key(stringcmp)):
44 54
         if not word:
45 55
             continue
56
+        word = word.encode('utf-8')
46 57
         for before, after in replace_list:
47
-            word = word.replace(before, after)
48
-        masked = masked.replace(word, STR_MASK)
49
-    return masked
58
+            word = word.replace(before.encode('utf-8'), after.encode('utf-8'))
59
+        masked = masked.replace(word, STR_MASK.encode('utf-8'))
60
+    return masked.decode('utf-8')
50 61
 
51 62
 
52 63
 def state_format(msg, state, color):

+ 1
- 1
packstack/modules/documentation.py View File

@@ -69,7 +69,7 @@ def update_params_usage(path, params, opt_title='OPTIONS', sectioned=True):
69 69
 
70 70
     if not _rst_cache:
71 71
         tree = core.publish_doctree(
72
-            source=open(path).read().decode('utf-8'), source_path=path
72
+            source=open(path).read(), source_path=path
73 73
         )
74 74
         for key, value in _iter_options(_get_options(tree, opt_title)):
75 75
             _rst_cache.setdefault(key, value)

+ 1
- 0
packstack/plugins/prescript_000.py View File

@@ -1160,6 +1160,7 @@ def manage_rdo(host, config):
1160 1160
     # yum-config-manager returns 0 always, but returns current setup
1161 1161
     # if succeeds
1162 1162
     rc, out = server.execute()
1163
+
1163 1164
     match = re.search('enabled\s*=\s*(1|True)', out)
1164 1165
     if not match:
1165 1166
         msg = ('Failed to set RDO repo on host %s:\nRPM file seems to be '

+ 3
- 3
tests/installer/test_sequences.py View File

@@ -15,8 +15,8 @@
15 15
 # License for the specific language governing permissions and limitations
16 16
 # under the License.
17 17
 
18
+import six
18 19
 import sys
19
-import StringIO
20 20
 from unittest import TestCase
21 21
 
22 22
 from packstack.installer import utils
@@ -29,7 +29,7 @@ class StepTestCase(PackstackTestCaseMixin, TestCase):
29 29
     def setUp(self):
30 30
         super(StepTestCase, self).setUp()
31 31
         self._stdout = sys.stdout
32
-        sys.stdout = StringIO.StringIO()
32
+        sys.stdout = six.StringIO()
33 33
 
34 34
     def tearDown(self):
35 35
         super(StepTestCase, self).tearDown()
@@ -57,7 +57,7 @@ class SequenceTestCase(PackstackTestCaseMixin, TestCase):
57 57
     def setUp(self):
58 58
         super(SequenceTestCase, self).setUp()
59 59
         self._stdout = sys.stdout
60
-        sys.stdout = StringIO.StringIO()
60
+        sys.stdout = six.StringIO()
61 61
 
62 62
         self.steps = [{'name': '1', 'function': lambda x, y: True,
63 63
                        'title': 'Step 1'},

+ 3
- 2
tests/installer/test_setup_params.py View File

@@ -19,6 +19,7 @@
19 19
 Test cases for packstack.installer.core.parameters module.
20 20
 """
21 21
 
22
+import six
22 23
 from unittest import TestCase
23 24
 
24 25
 from ..test_base import PackstackTestCaseMixin
@@ -49,7 +50,7 @@ class ParameterTestCase(PackstackTestCaseMixin, TestCase):
49 50
         initialization
50 51
         """
51 52
         param = Parameter(self.data)
52
-        for key, value in self.data.iteritems():
53
+        for key, value in six.iteritems(self.data):
53 54
             self.assertEqual(getattr(param, key), value)
54 55
 
55 56
     def test_default_attribute(self):
@@ -80,7 +81,7 @@ class GroupTestCase(PackstackTestCaseMixin, TestCase):
80 81
         Test packstack.installer.core.parameters.Group initialization
81 82
         """
82 83
         group = Group(attributes=self.attrs, parameters=self.params)
83
-        for key, value in self.attrs.iteritems():
84
+        for key, value in six.iteritems(self.attrs):
84 85
             self.assertEqual(getattr(group, key), value)
85 86
         for param in self.params:
86 87
             self.assertIn(param['CONF_NAME'], group.parameters)

+ 5
- 4
tests/test_base.py View File

@@ -49,7 +49,8 @@ class FakePopen(object):
49 49
         '''Register a fake script.'''
50 50
         if isinstance(args, list):
51 51
             args = '\n'.join(args)
52
-        prefix = "function t(){ exit $? ; } \n trap t ERR \n"
52
+
53
+        prefix = "function t(){ exit $? ; } \n trap t ERR \n "
53 54
         args = prefix + args
54 55
         cls.script_registry[args] = {'stdout': stdout,
55 56
                                      'stderr': stderr,
@@ -86,8 +87,8 @@ class FakePopen(object):
86 87
 
87 88
     def communicate(self, input=None):
88 89
         if self._is_script:
89
-            if input in self.script_registry:
90
-                this = self.script_registry[input]
90
+            if input.decode('utf-8') in self.script_registry:
91
+                this = self.script_registry[input.decode('utf-8')]
91 92
             else:
92 93
                 LOG.warning('call to unregistered script: %s', input)
93 94
                 this = {'stdout': '', 'stderr': '', 'returncode': 0}
@@ -128,7 +129,7 @@ class PackstackTestCaseMixin(object):
128 129
                 raise AssertionError(_msg)
129 130
 
130 131
     def assertListEqual(self, list1, list2, msg=None):
131
-        f, s = len(list1), len(list2)
132
+        f, s = len(list(list1)), len(list(list2))
132 133
         _msg = msg or ('Element counts were not equal. First has %s, '
133 134
                        'Second has %s' % (f, s))
134 135
         self.assertEqual(f, s, msg=_msg)

+ 1
- 1
tox.ini View File

@@ -1,6 +1,6 @@
1 1
 [tox]
2 2
 minversion = 1.6
3
-envlist = py27,pep8,releasenotes
3
+envlist = py27,py36,pep8,releasenotes
4 4
 skipsdist = True
5 5
 
6 6
 [testenv]

Loading…
Cancel
Save