Ensure to decode bytes or fail

The commit 048316e981 introduces the
pattern:

  if isinstance(line, bytes):
    try:
      line = line.decode(encoding='utf-8')
    except UnicodeError:
      pass
  # concat line with a string

which is not working in PY3K if an UnicodeError is raised because line
is (silently) not decoded and concatened to a string.

This change ensures to return a text object or to raise an error.

Closes-Bug: #1503415
Blueprint: neutron-python3
Change-Id: I16b8013f33aa3efad65be8040d3210120e047bbd

neutron/agent/linux/utils.py

@@ -123,15 +123,9 @@ def execute(cmd, process_input=None, addl_env=None,
             obj.stdin.close()
         if six.PY3:
             if isinstance(_stdout, bytes):
-                try:
-                    _stdout = _stdout.decode(encoding='utf-8')
-                except UnicodeError:
-                    pass
+                _stdout = _stdout.decode('utf-8', 'surrogateescape')
             if isinstance(_stderr, bytes):
-                try:
-                    _stderr = _stderr.decode(encoding='utf-8')
-                except UnicodeError:
-                    pass
+                _stderr = _stderr.decode('utf-8', 'surrogateescape')
 
         command_str = {
             'cmd': cmd,

neutron/agent/windows/utils.py

@@ -59,15 +59,9 @@ def execute(cmd, process_input=None, addl_env=None,
         obj.stdin.close()
         if six.PY3:
             if isinstance(_stdout, bytes):
-                try:
-                    _stdout = _stdout.decode(encoding='utf-8')
-                except UnicodeError:
-                    pass
+                _stdout = _stdout.decode('utf-8', 'surrogateescape')
             if isinstance(_stderr, bytes):
-                try:
-                    _stderr = _stderr.decode(encoding='utf-8')
-                except UnicodeError:
-                    pass
+                _stderr = _stderr.decode('utf-8', 'surrogateescape')
 
         m = _("\nCommand: %(cmd)s\nExit code: %(code)s\nStdin: %(stdin)s\n"
               "Stdout: %(stdout)s\nStderr: %(stderr)s") % \

neutron/tests/unit/agent/linux/test_utils.py

@@ -147,20 +147,15 @@ class AgentUtilsExecuteTest(base.BaseTestCase):
         result = utils.execute(['ls', self.test_file], return_stderr=True)
         self.assertEqual((str_data, ''), result)
 
-    def test_raise_unicodeerror_in_decoding_out_data(self):
-        class m_bytes(bytes):
-            def decode(self, encoding=None):
-                raise UnicodeError
-
-        err_data = 'UnicodeError'
-        bytes_err_data = b'UnicodeError'
+    @testtools.skipUnless(six.PY3, 'This test makes sense only in Python 3')
+    def test_surrogateescape_in_decoding_out_data(self):
+        bytes_err_data = b'\xed\xa0\xbd'
+        err_data = bytes_err_data.decode('utf-8', 'surrogateescape')
         out_data = "%s\n" % self.test_file
-        bytes_out_data = m_bytes(out_data.encode(encoding='utf-8'))
-        if six.PY3:
-            self.mock_popen.return_value = [bytes_out_data, bytes_err_data]
-            result = utils.execute(['ls', self.test_file],
-                                   return_stderr=True)
-            self.assertEqual((bytes_out_data, err_data), result)
+        bytes_out_data = out_data.encode(encoding='utf-8')
+        self.mock_popen.return_value = [bytes_out_data, bytes_err_data]
+        result = utils.execute(['ls', self.test_file], return_stderr=True)
+        self.assertEqual((out_data, err_data), result)
 
 
 class AgentUtilsExecuteEncodeTest(base.BaseTestCase):