Browse Source

[task] Fix verification log

While fix for bug #1562713 changed the way how to save verification log in
case of some issues at CLI layer, it did not fixed old saved logs. Also, it
produced one more bug - IOError trace was saved as list instead of string.

This patch fixes the way of storage IOError and add migration to fix old data

Also, we change format of verification_log to dict here
Change-Id: Iffcf066e7033acd48f1446baebe0b2e9a334449f
Related-Bug: #1562713
tags/0.7.0
Andrey Kurilin 2 years ago
parent
commit
2bfe9a9928

+ 5
- 5
rally/cli/commands/task.py View File

@@ -138,7 +138,7 @@ class TaskCommands(object):
138 138
             if task_instance:
139 139
                 task_instance.set_failed(err.__class__.__name__,
140 140
                                          str(err),
141
-                                         json.dumps(traceback.format_stack()))
141
+                                         json.dumps(traceback.format_exc()))
142 142
             raise
143 143
         api.Task.validate(deployment, input_task, task_instance)
144 144
         print(_("Task config is valid :)"))
@@ -325,11 +325,11 @@ class TaskCommands(object):
325 325
             print("-" * 80)
326 326
             verification = yaml.safe_load(task["verification_log"])
327 327
             if logging.is_debug():
328
-                print(yaml.safe_load(verification[2]))
328
+                print(yaml.safe_load(verification["trace"]))
329 329
             else:
330
-                print(verification[0])
331
-                print(verification[1])
332
-                print(_("\nFor more details run:\nrally -vd task detailed %s")
330
+                print(verification["etype"])
331
+                print(verification["msg"])
332
+                print(_("\nFor more details run:\nrally -d task detailed %s")
333 333
                       % task["uuid"])
334 334
             return 0
335 335
         elif task["status"] not in [consts.TaskStatus.FINISHED,

+ 18
- 1
rally/common/db/sqlalchemy/migrations/script.py.mako View File

@@ -1,3 +1,17 @@
1
+# All Rights Reserved.
2
+#
3
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+#    not use this file except in compliance with the License. You may obtain
5
+#    a copy of the License at
6
+#
7
+#         http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+#    Unless required by applicable law or agreed to in writing, software
10
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+#    License for the specific language governing permissions and limitations
13
+#    under the License.
14
+
1 15
 """${message}
2 16
 
3 17
 Revision ID: ${up_revision}
@@ -16,9 +30,12 @@ from alembic import op
16 30
 import sqlalchemy as sa
17 31
 ${imports if imports else ""}
18 32
 
33
+${"from rally import exceptions" if not downgrades else ""}
34
+
35
+
19 36
 def upgrade():
20 37
     ${upgrades if upgrades else "pass"}
21 38
 
22 39
 
23 40
 def downgrade():
24
-    ${downgrades if downgrades else "pass"}
41
+    ${downgrades if downgrades else "raise exceptions.DowngradeNotSupported()"}

+ 112
- 0
rally/common/db/sqlalchemy/migrations/versions/08e1515a576c_fix_invalid_verification_logs.py View File

@@ -0,0 +1,112 @@
1
+# All Rights Reserved.
2
+#
3
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+#    not use this file except in compliance with the License. You may obtain
5
+#    a copy of the License at
6
+#
7
+#         http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+#    Unless required by applicable law or agreed to in writing, software
10
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+#    License for the specific language governing permissions and limitations
13
+#    under the License.
14
+
15
+"""fix invalid verification logs
16
+
17
+Revision ID: 08e1515a576c
18
+Revises: 54e844ebfbc3
19
+Create Date: 2016-09-12 15:47:11.279610
20
+
21
+"""
22
+
23
+# revision identifiers, used by Alembic.
24
+revision = "08e1515a576c"
25
+down_revision = "54e844ebfbc3"
26
+branch_labels = None
27
+depends_on = None
28
+
29
+
30
+import json
31
+import uuid
32
+
33
+from alembic import op
34
+import sqlalchemy as sa
35
+
36
+from rally import consts
37
+from rally import exceptions
38
+
39
+
40
+def UUID():
41
+    return str(uuid.uuid4())
42
+
43
+
44
+task_helper = sa.Table(
45
+    "tasks",
46
+    sa.MetaData(),
47
+    sa.Column("id", sa.Integer, primary_key=True, autoincrement=True),
48
+    sa.Column("uuid", sa.String(36), default=UUID, nullable=False),
49
+    sa.Column("status", sa.Enum(*list(consts.TaskStatus),
50
+                                name="enum_tasks_status"),
51
+              default=consts.TaskStatus.INIT, nullable=False),
52
+    sa.Column("verification_log", sa.Text, default=""),
53
+    sa.Column("tag", sa.String(64), default=""),
54
+    sa.Column("deployment_uuid", sa.String(36), nullable=False)
55
+)
56
+
57
+
58
+def _make_trace(etype, emsg, raw_trace=None):
59
+    trace = "Traceback (most recent call last):\n"
60
+    if raw_trace is None:
61
+        trace += "\n\t\t...n/a..\n\n"
62
+    else:
63
+        trace += "".join(json.loads(raw_trace))
64
+
65
+    trace += "%s: %s" % (etype, emsg)
66
+    return trace
67
+
68
+
69
+def upgrade():
70
+    connection = op.get_bind()
71
+    for task in connection.execute(task_helper.select()):
72
+        verification_log = task.verification_log
73
+
74
+        if not verification_log:
75
+            continue
76
+
77
+        new_value = None
78
+
79
+        verification_log = json.loads(verification_log)
80
+        if isinstance(verification_log, list):
81
+            new_value = {"etype": verification_log[0],
82
+                         "msg": verification_log[1],
83
+                         "trace": verification_log[2]}
84
+            if new_value["trace"].startswith("["):
85
+                # NOTE(andreykurilin): For several cases traceback was
86
+                #   transmitted as list instead of string.
87
+                new_value["trace"] = _make_trace(*verification_log)
88
+        else:
89
+            if verification_log.startswith("No such file"):
90
+                new_value = {"etype": IOError.__name__,
91
+                             "msg": verification_log}
92
+                new_value["trace"] = _make_trace(new_value["etype"],
93
+                                                 new_value["msg"])
94
+            elif verification_log.startswith("Task config is invalid"):
95
+                new_value = {"etype": exceptions.InvalidTaskException.__name__,
96
+                             "msg": verification_log}
97
+                new_value["trace"] = _make_trace(new_value["etype"],
98
+                                                 new_value["msg"])
99
+            elif verification_log.startswith("Failed to load task"):
100
+                new_value = {"etype": "FailedToLoadTask",
101
+                             "msg": verification_log}
102
+                new_value["trace"] = _make_trace(new_value["etype"],
103
+                                                 new_value["msg"])
104
+
105
+        if new_value:
106
+            connection.execute(task_helper.update().where(
107
+                task_helper.c.id == task.id).values(
108
+                verification_log=json.dumps(new_value)))
109
+
110
+
111
+def downgrade():
112
+    raise exceptions.DowngradeNotSupported()

+ 2
- 4
rally/common/objects/task.py View File

@@ -350,10 +350,8 @@ class Task(object):
350 350
 
351 351
     def set_failed(self, etype, msg, etraceback):
352 352
         self._update({"status": consts.TaskStatus.FAILED,
353
-                      "verification_log": json.dumps([etype,
354
-                                                      msg,
355
-                                                      etraceback
356
-                                                      ])})
353
+                      "verification_log": json.dumps(
354
+                          {"etype": etype, "msg": msg, "trace": etraceback})})
357 355
 
358 356
     def get_results(self):
359 357
         return db.task_result_get_all_by_uuid(self.task["uuid"])

+ 7
- 6
tests/unit/cli/commands/test_task.py View File

@@ -324,8 +324,9 @@ class TaskCommandsTestCase(test.TestCase):
324 324
             "uuid": test_uuid,
325 325
             "status": consts.TaskStatus.FAILED,
326 326
             "results": [],
327
-            "verification_log": json.dumps(["error_type", "error_message",
328
-                                            "error_traceback"])
327
+            "verification_log": json.dumps({"etype": "error_type",
328
+                                            "msg": "error_message",
329
+                                            "trace": "error_traceback"})
329 330
         }
330 331
         mock_task.get_detailed = mock.MagicMock(return_value=value)
331 332
 
@@ -334,14 +335,14 @@ class TaskCommandsTestCase(test.TestCase):
334 335
         verification = yaml.safe_load(value["verification_log"])
335 336
         if debug:
336 337
             expected_calls = [mock.call("Task test_task_id: failed"),
337
-                              mock.call("%s" % verification[2])]
338
+                              mock.call("%s" % verification["trace"])]
338 339
             mock_stdout.write.assert_has_calls(expected_calls, any_order=True)
339 340
         else:
340 341
             expected_calls = [mock.call("Task test_task_id: failed"),
341
-                              mock.call("%s" % verification[0]),
342
-                              mock.call("%s" % verification[1]),
342
+                              mock.call("%s" % verification["etype"]),
343
+                              mock.call("%s" % verification["msg"]),
343 344
                               mock.call("\nFor more details run:\nrally "
344
-                                        "-vd task detailed %s" % test_uuid)]
345
+                                        "-d task detailed %s" % test_uuid)]
345 346
             mock_stdout.write.assert_has_calls(expected_calls, any_order=True)
346 347
 
347 348
     @mock.patch("rally.cli.commands.task.api.Task")

+ 73
- 0
tests/unit/common/db/test_migrations.py View File

@@ -383,3 +383,76 @@ class MigrationWalkTestCase(rtest.DBTestCase,
383 383
                     deployment_table.delete().where(
384 384
                         deployment_table.c.uuid == deployment.uuid)
385 385
                 )
386
+
387
+    def _pre_upgrade_08e1515a576c(self, engine):
388
+        self._08e1515a576c_logs = [
389
+            {"pre": "No such file name",
390
+             "post": {"etype": IOError.__name__, "msg": "No such file name"}},
391
+            {"pre": "Task config is invalid: bla",
392
+             "post": {"etype": "InvalidTaskException",
393
+                      "msg": "Task config is invalid: bla"}},
394
+            {"pre": "Failed to load task foo",
395
+             "post": {"etype": "FailedToLoadTask",
396
+                      "msg": "Failed to load task foo"}},
397
+            {"pre": ["SomeCls", "msg", json.dumps(
398
+                ["File some1.py, line ...\n",
399
+                 "File some2.py, line ...\n"])],
400
+             "post": {"etype": "SomeCls",
401
+                      "msg": "msg",
402
+                      "trace": "Traceback (most recent call last):\n"
403
+                               "File some1.py, line ...\n"
404
+                               "File some2.py, line ...\nSomeCls: msg"}},
405
+        ]
406
+
407
+        deployment_table = db_utils.get_table(engine, "deployments")
408
+        task_table = db_utils.get_table(engine, "tasks")
409
+
410
+        self._08e1515a576c_deployment_uuid = "08e1515a576c-uuuu-uuuu-iiii-dddd"
411
+        with engine.connect() as conn:
412
+            conn.execute(
413
+                deployment_table.insert(),
414
+                [{"uuid": self._08e1515a576c_deployment_uuid,
415
+                  "name": self._08e1515a576c_deployment_uuid,
416
+                  "config": six.b("{}"),
417
+                  "enum_deployments_status":
418
+                      consts.DeployStatus.DEPLOY_FINISHED,
419
+                  "credentials": six.b(json.dumps([])),
420
+                  "users": six.b(json.dumps([]))
421
+                  }])
422
+            for i in range(0, len(self._08e1515a576c_logs)):
423
+                log = json.dumps(self._08e1515a576c_logs[i]["pre"])
424
+                conn.execute(
425
+                    task_table.insert(),
426
+                    [{"uuid": i,
427
+                      "verification_log": log,
428
+                      "status": consts.TaskStatus.FAILED,
429
+                      "enum_tasks_status": consts.TaskStatus.FAILED,
430
+                      "deployment_uuid": self._08e1515a576c_deployment_uuid
431
+                      }])
432
+
433
+    def _check_08e1515a576c(self, engine, data):
434
+        self.assertEqual("08e1515a576c",
435
+                         api.get_backend().schema_revision(engine=engine))
436
+
437
+        tasks = self._08e1515a576c_logs
438
+
439
+        deployment_table = db_utils.get_table(engine, "deployments")
440
+        task_table = db_utils.get_table(engine, "tasks")
441
+
442
+        with engine.connect() as conn:
443
+            tasks_found = conn.execute(task_table.select()).fetchall()
444
+            for task in tasks_found:
445
+                actual_log = json.loads(task.verification_log)
446
+                self.assertIsInstance(actual_log, dict)
447
+                expected = tasks[int(task.uuid)]["post"]
448
+                for key in expected:
449
+                    self.assertEqual(expected[key], actual_log[key])
450
+
451
+                conn.execute(
452
+                    task_table.delete().where(task_table.c.uuid == task.uuid))
453
+
454
+            deployment_uuid = self._08e1515a576c_deployment_uuid
455
+            conn.execute(
456
+                deployment_table.delete().where(
457
+                    deployment_table.c.uuid == deployment_uuid)
458
+            )

+ 3
- 2
tests/unit/common/objects/test_task.py View File

@@ -249,8 +249,9 @@ class TaskTestCase(test.TestCase):
249 249
         mock_task_update.assert_called_once_with(
250 250
             self.task["uuid"],
251 251
             {"status": consts.TaskStatus.FAILED,
252
-             "verification_log": "[\"foo_type\", \"foo_error_message\", "
253
-                                 "\"foo_trace\"]"},
252
+             "verification_log": json.dumps({"etype": "foo_type",
253
+                                             "msg": "foo_error_message",
254
+                                             "trace": "foo_trace"})},
254 255
         )
255 256
 
256 257
     @ddt.data(

Loading…
Cancel
Save