Browse Source

clean up

Change-Id: I6707be5c6f4f88f72ef843ba13947ab54824ca32
Kanagaraj Manickam 3 years ago
parent
commit
1f8b1e2f04
5 changed files with 116 additions and 88 deletions
  1. 20
    3
      namos/cmd/manage.py
  2. 14
    44
      namos/conductor/manager.py
  3. 4
    4
      namos/conductor/rpcapi.py
  4. 6
    2
      namos/db/api.py
  5. 72
    35
      namos/db/sqlalchemy/api.py

+ 20
- 3
namos/cmd/manage.py View File

@@ -30,6 +30,9 @@ MANAGE_COMMAND_NAME = 'namos-manage'
30 30
 
31 31
 
32 32
 class HeartBeat(object):
33
+    def cleanup(self):
34
+        api.cleanup(None)
35
+
33 36
     def report_status(self):
34 37
         print_format = "%-20s%-15s%-15s%-35s%-10s"
35 38
         strip = 90 * '-'
@@ -40,12 +43,19 @@ class HeartBeat(object):
40 43
                               'Component',
41 44
                               'Status'))
42 45
         print(strip)
43
-        for k, s in api.get_status(None).items():
46
+        for k, s in api.get_status(
47
+            None,
48
+            CONF.command.node,
49
+            CONF.command.service,
50
+            CONF.command.type,
51
+            CONF.command.component
52
+        ).items():
53
+            # if not CONF.command.component and s['is_launcher']:
44 54
             print(print_format % (s['node'],
45 55
                                   s['type'],
46 56
                                   s['service'],
47
-                                  s['component'],
48
-                                  s['status']))
57
+                                  s['worker'],
58
+                                  ':)' if s['status'] else 'XXX'))
49 59
         print(strip)
50 60
 
51 61
 
@@ -193,8 +203,15 @@ def add_command_parsers(subparsers):
193 203
     parser.set_defaults(func=OsloConfigSchemaManager().sync)
194 204
 
195 205
     parser = subparsers.add_parser('status')
206
+    parser.add_argument('-n', '--node')
207
+    parser.add_argument('-s', '--service')
208
+    parser.add_argument('-c', '--component')
209
+    parser.add_argument('-t', '--type')
196 210
     parser.set_defaults(func=HeartBeat().report_status)
197 211
 
212
+    parser = subparsers.add_parser('cleanup')
213
+    parser.set_defaults(func=HeartBeat().cleanup)
214
+
198 215
 command_opt = cfg.SubCommandOpt('command',
199 216
                                 title='Command',
200 217
                                 help='Available commands',

+ 14
- 44
namos/conductor/manager.py View File

@@ -69,7 +69,7 @@ class ConductorManager(object):
69 69
         return db_api.region_delete(context. region_id)
70 70
 
71 71
     @request_context
72
-    def add_service_node(self, context, service_node):
72
+    def service_node_create(self, context, service_node):
73 73
         return db_api.service_node_create(context, service_node)
74 74
 
75 75
     @request_context
@@ -137,12 +137,12 @@ class ConductorManager(object):
137 137
 
138 138
         # TODO(mrkanag) Move this to periofic task, before deleting each
139 139
         # sw, make usre its created atleast 5 mins before
140
-        # sp.cleanup(service_component_id)
140
+        sp.cleanup(service_component_id)
141 141
         return service_worker_id
142 142
 
143 143
     def _regisgration_ackw(self, context, identification):
144 144
         client = messaging.get_rpc_client(
145
-            topic=self.os_namos_listener_topic(identification),
145
+            topic=self._os_namos_listener_topic(identification),
146 146
             version=self.RPC_API_VERSION,
147 147
             exchange=namos_config.PROJECT_NAME)
148 148
         client.cast(context,
@@ -150,12 +150,12 @@ class ConductorManager(object):
150 150
                     identification=identification)
151 151
         LOG.info("REGISTER [%s] ACK" % identification)
152 152
 
153
-    def os_namos_listener_topic(self, identification):
153
+    def _os_namos_listener_topic(self, identification):
154 154
         return 'namos.CONF.%s' % identification
155 155
 
156 156
     def _ping(self, context, identification):
157 157
         client = messaging.get_rpc_client(
158
-            topic=self.os_namos_listener_topic(identification),
158
+            topic=self._os_namos_listener_topic(identification),
159 159
             version=self.RPC_API_VERSION,
160 160
             exchange=namos_config.PROJECT_NAME,
161 161
             timeout=1)
@@ -170,9 +170,9 @@ class ConductorManager(object):
170 170
             LOG.info("PING [%s] FAILED" % identification)
171 171
             return False
172 172
 
173
-    def update_config_file(self, context, identification, name, content):
173
+    def _update_config_file(self, context, identification, name, content):
174 174
         client = messaging.get_rpc_client(
175
-            topic=self.os_namos_listener_topic(identification),
175
+            topic=self._os_namos_listener_topic(identification),
176 176
             version=self.RPC_API_VERSION,
177 177
             exchange=namos_config.PROJECT_NAME,
178 178
             timeout=2)
@@ -304,10 +304,10 @@ class ConductorManager(object):
304 304
                         # TODO(mrkanag) is ping() better option instead?
305 305
                         if utils.find_status(sw):
306 306
                             try:
307
-                                self.update_config_file(context,
308
-                                                        sw.pid,
309
-                                                        cf.name,
310
-                                                        cf.file)
307
+                                self._update_config_file(context,
308
+                                                         sw.pid,
309
+                                                         cf.name,
310
+                                                         cf.file)
311 311
                                 cf['status'] = 'completed'
312 312
                                 return cf
313 313
                             except:  # noqa
@@ -448,8 +448,8 @@ class ServiceProcessor(object):
448 448
             service_worker = db_api.service_worker_create(
449 449
                 self.context,
450 450
                 # TODO(mrkanag) Fix the name, device driver proper !
451
-                dict(name='%s@%s' % (self.registration_info['pid'],
452
-                                     service_component.name),
451
+                dict(name='%s@%s' % (service_component.name,
452
+                                     self.registration_info['pid']),
453 453
                      pid=self.registration_info['identification'],
454 454
                      host=self.registration_info['host'],
455 455
                      service_component_id=service_component.id,
@@ -470,37 +470,7 @@ class ServiceProcessor(object):
470 470
 
471 471
     def cleanup(self, service_component_id):
472 472
         # clean up the dead service workers
473
-        #  TODO(mrkanag) Make this into thread
474
-        service_workers = \
475
-            db_api.service_worker_get_all_by(
476
-                self.context,
477
-                service_component_id=service_component_id
478
-            )
479
-
480
-        for srv_wkr in service_workers:
481
-            # TODO(mrkanag) Move this to db layer and query non deleted entries
482
-            if srv_wkr.deleted_at is not None:
483
-                continue
484
-
485
-            # TODO(mrkanag) is this interval ok
486
-            if utils.find_status(srv_wkr, report_interval=60):
487
-                LOG.info('Service Worker %s is live'
488
-                         % srv_wkr.id)
489
-                continue
490
-            else:
491
-                confs = db_api.config_get_by_name_for_service_worker(
492
-                    self.context,
493
-                    service_worker_id=srv_wkr.id
494
-                )
495
-
496
-                for conf in confs:
497
-                    db_api.config_delete(self.context, conf.id)
498
-                    LOG.info('Config %s is deleted'
499
-                             % conf.id)
500
-
501
-                db_api.service_worker_delete(self.context, srv_wkr.id)
502
-                LOG.info('Service Worker %s is deleted'
503
-                         % srv_wkr.id)
473
+        db_api.cleanup(self.context, service_component_id)
504 474
 
505 475
 
506 476
 class ConfigProcessor(object):

+ 4
- 4
namos/conductor/rpcapi.py View File

@@ -95,7 +95,7 @@ class ConductorAPI(object):
95 95
             region_id=region_id)
96 96
 
97 97
     @wrapper_function
98
-    def add_service_node(self, context, service_node):
98
+    def service_node_create(self, context, service_node):
99 99
         return self.client.call(
100 100
             context,
101 101
             'service_node_create',
@@ -248,6 +248,6 @@ if __name__ == '__main__':
248 248
                                         'dcf0f17b-99f6-49e9-8d5f-23b3ad1167dc',
249 249
                                         content))
250 250
 
251
-    # print_config_schema()
252
-    # print_view_360()
253
-    sample_config_update()
251
+    print_config_schema()
252
+    print_view_360()
253
+    # sample_config_update()

+ 6
- 2
namos/db/api.py View File

@@ -484,5 +484,9 @@ def view_360(context, include_conf_file=False, include_status=False):
484 484
                          include_status=include_status)
485 485
 
486 486
 
487
-def get_status(context):
488
-    return IMPL.get_status(context)
487
+def get_status(context, node=None, service=None, type=None, component=None):
488
+    return IMPL.get_status(context, node, service, type, component)
489
+
490
+
491
+def cleanup(context, service_component_id=None, dead_since=300):
492
+    return IMPL.cleanup(context, service_component_id, dead_since)

+ 72
- 35
namos/db/sqlalchemy/api.py View File

@@ -175,7 +175,7 @@ def device_get_all(context):
175 175
     return _get_all(context, models.Device)
176 176
 
177 177
 
178
-def _device_get_all_by(context, **kwargs):
178
+def device_get_all_by(context, **kwargs):
179 179
     return _get_all_by(context, models.Device, **kwargs)
180 180
 
181 181
 
@@ -227,7 +227,7 @@ def device_endpoint_get_all(context):
227 227
     return _get_all(context, models.DeviceEndpoint)
228 228
 
229 229
 
230
-def _device_endpoint_get_all_by(context, **kwargs):
230
+def device_endpoint_get_all_by(context, **kwargs):
231 231
     return _get_all_by(context, models.DeviceEndpoint, **kwargs)
232 232
 
233 233
 
@@ -282,7 +282,7 @@ def device_driver_get_all(context):
282 282
     return _get_all(context, models.DeviceDriver)
283 283
 
284 284
 
285
-def _device_driver_get_all_by(context, **kwargs):
285
+def device_driver_get_all_by(context, **kwargs):
286 286
     return _get_all_by(context, models.DeviceDriver, **kwargs)
287 287
 
288 288
 
@@ -320,7 +320,7 @@ def device_driver_class_get_all(context):
320 320
     return _get_all(context, models.DeviceDriverClass)
321 321
 
322 322
 
323
-def _device_driver_classget_all_by(context, **kwargs):
323
+def device_driver_class_get_all_by(context, **kwargs):
324 324
     return _get_all_by(context, models.DeviceDriverClass, **kwargs)
325 325
 
326 326
 
@@ -358,7 +358,7 @@ def service_get_all(context):
358 358
     return _get_all(context, models.Service)
359 359
 
360 360
 
361
-def _service_get_all_by(context, **kwargs):
361
+def service_get_all_by(context, **kwargs):
362 362
     return _get_all_by(context, models.Service, **kwargs)
363 363
 
364 364
 
@@ -396,7 +396,7 @@ def service_node_get_all(context):
396 396
     return _get_all(context, models.ServiceNode)
397 397
 
398 398
 
399
-def _service_node_get_all_by(context, **kwargs):
399
+def service_node_get_all_by(context, **kwargs):
400 400
     return _get_all_by(context, models.ServiceNode, **kwargs)
401 401
 
402 402
 
@@ -460,7 +460,7 @@ def service_component_get_all(context):
460 460
     return _get_all(context, models.ServiceComponent)
461 461
 
462 462
 
463
-def _service_component_get_all_by(context, **kwargs):
463
+def service_component_get_all_by(context, **kwargs):
464 464
     return _get_all_by(context, models.ServiceComponent, **kwargs)
465 465
 
466 466
 
@@ -510,10 +510,6 @@ def service_worker_get_all(context):
510 510
 
511 511
 
512 512
 def service_worker_get_all_by(context, **kwargs):
513
-    return _service_worker_get_all_by(context, **kwargs)
514
-
515
-
516
-def _service_worker_get_all_by(context, **kwargs):
517 513
     return _get_all_by(context, models.ServiceWorker, **kwargs)
518 514
 
519 515
 
@@ -547,7 +543,6 @@ def config_schema_get_by_name(context, name):
547 543
     return config
548 544
 
549 545
 
550
-# TODO(mrkanag) fix it to take **kwargs
551 546
 def config_schema_get_by(context,
552 547
                          namespace=None,
553 548
                          group=None,
@@ -622,7 +617,7 @@ def config_get_all(context):
622 617
     return _get_all(context, models.OsloConfig)
623 618
 
624 619
 
625
-def _config_get_all_by(context, **kwargs):
620
+def config_get_all_by(context, **kwargs):
626 621
     return _get_all_by(context, models.OsloConfig, **kwargs)
627 622
 
628 623
 
@@ -721,7 +716,7 @@ def config_file_get_all(context):
721 716
     return _get_all(context, models.OsloConfigFile)
722 717
 
723 718
 
724
-def _config_file_get_all_by(context, **kwargs):
719
+def config_file_get_all_by(context, **kwargs):
725 720
     return _get_all_by(context, models.OsloConfigFile, **kwargs)
726 721
 
727 722
 
@@ -743,8 +738,8 @@ def service_perspective_get(context, service_id, include_details=False):
743 738
     # on include_details, for each of the entity, include complete details
744 739
     service_perspective = dict()
745 740
     service_perspective['service'] = service_get(context, service_id).to_dict()
746
-    service_components = _service_component_get_all_by(context,
747
-                                                       service_id=service_id)
741
+    service_components = service_component_get_all_by(context,
742
+                                                      service_id=service_id)
748 743
     service_perspective['service_components'] = dict()
749 744
     # service_perspective['service_components']['size'] =
750 745
     # len(service_components)
@@ -755,7 +750,7 @@ def service_perspective_get(context, service_id, include_details=False):
755 750
             = sc.to_dict()
756 751
         service_perspective['service_components'][sc.id]['service_node']\
757 752
             = service_node_get(context, sc.node_id).to_dict()
758
-        service_workers = _service_worker_get_all_by(
753
+        service_workers = service_worker_get_all_by(
759 754
             context,
760 755
             service_component_id=sc.id)
761 756
         service_perspective['service_components'][sc.id]['service_workers'] \
@@ -770,7 +765,7 @@ def service_perspective_get(context, service_id, include_details=False):
770 765
                 sc.id]['service_workers'][sw.id][
771 766
                 'service_worker'] = sw.to_dict()
772 767
 
773
-            device_drivers = _device_driver_get_all_by(
768
+            device_drivers = device_driver_get_all_by(
774 769
                 context,
775 770
                 service_worker_id=sw.id)
776 771
             service_perspective['service_components'][
@@ -819,8 +814,8 @@ def device_perspective_get(context, device_id, include_details=False):
819 814
     # on include_details, for each of the entity, include complete details
820 815
     device_perspective = dict()
821 816
     device_perspective['device'] = device_get(context, device_id).to_dict()
822
-    endpoints = _device_endpoint_get_all_by(context,
823
-                                            device_id=device_id)
817
+    endpoints = device_endpoint_get_all_by(context,
818
+                                           device_id=device_id)
824 819
     device_perspective['device_endpoints'] = dict()
825 820
     # device_perspective['device_endpoints']['size'] = len(endpoints)
826 821
 
@@ -829,8 +824,8 @@ def device_perspective_get(context, device_id, include_details=False):
829 824
         device_perspective['device_endpoints'][
830 825
             ep.id]['device_endpoint'] = ep.to_dict()
831 826
 
832
-        device_drivers = _device_driver_get_all_by(context,
833
-                                                   endpoint_id=ep.id)
827
+        device_drivers = device_driver_get_all_by(context,
828
+                                                  endpoint_id=ep.id)
834 829
         device_perspective['device_endpoints'][
835 830
             ep.id]['device_drivers'] = dict()
836 831
         # device_perspective['device_endpoints'][ep.id] \
@@ -881,8 +876,8 @@ def region_perspective_get(context, region_id, include_details=False):
881 876
 
882 877
     region_perspective = dict()
883 878
     region_perspective['region'] = region_get(context, region_id).to_dict()
884
-    s_nodes = _service_node_get_all_by(context,
885
-                                       region_id=region_id)
879
+    s_nodes = service_node_get_all_by(context,
880
+                                      region_id=region_id)
886 881
     # region_perspective['service_nodes'] = dict()
887 882
     # region_perspective['service_nodes']['size'] = len(s_nodes)
888 883
     # for s_node in s_nodes:
@@ -906,7 +901,7 @@ def region_perspective_get(context, region_id, include_details=False):
906 901
 
907 902
     region_perspective['services'] = dict()
908 903
     for s_node in s_nodes:
909
-        s_components = _service_component_get_all_by(
904
+        s_components = service_component_get_all_by(
910 905
             context,
911 906
             node_id=s_node.id)
912 907
         srvs = list()
@@ -920,7 +915,7 @@ def region_perspective_get(context, region_id, include_details=False):
920 915
             s = service_get(context, s_id)
921 916
             region_perspective['services'][s_id] = s.to_dict()
922 917
 
923
-    devices = _device_get_all_by(context, region_id=region_id)
918
+    devices = device_get_all_by(context, region_id=region_id)
924 919
     region_perspective['devices'] = dict()
925 920
     # region_perspective['devices']['size'] = len(devices)
926 921
     for d in devices:
@@ -980,8 +975,8 @@ def view_360(context, include_conf_file=False, include_status=False):
980 975
         view['region'][rg.id] = region_get(context, rg.id).to_dict()
981 976
 
982 977
         view['region'][rg.id]['service_node'] = dict()
983
-        srv_nd_lst = _service_node_get_all_by(context,
984
-                                              region_id=rg.id)
978
+        srv_nd_lst = service_node_get_all_by(context,
979
+                                             region_id=rg.id)
985 980
         for srv_nd in srv_nd_lst:
986 981
             # service node
987 982
             view['service_node'][srv_nd.id] = service_node_get(
@@ -1051,7 +1046,7 @@ def view_360(context, include_conf_file=False, include_status=False):
1051 1046
                     view['region'][rg.id]['service_node'][srv_nd.id][
1052 1047
                         'service_component'][srv_cmp.id][
1053 1048
                         'service_worker'][srv_wkr.id]['device_driver'] = dict()
1054
-                    dvc_drv_list = _device_driver_get_all_by(
1049
+                    dvc_drv_list = device_driver_get_all_by(
1055 1050
                         context,
1056 1051
                         service_worker_id=srv_wkr.id
1057 1052
                     )
@@ -1111,33 +1106,75 @@ def view_360(context, include_conf_file=False, include_status=False):
1111 1106
     return view
1112 1107
 
1113 1108
 
1114
-def get_status(context):
1109
+def get_status(context, node=None, service=None, type=None, component=None):
1115 1110
     sr = {}
1116 1111
     for sn in service_node_get_all(context):
1112
+        if node and not node == sn.name:
1113
+            continue
1114
+
1117 1115
         for sc in service_component_get_all_by_node_for_service(
1118 1116
             context,
1119 1117
             node_id=sn.id
1120 1118
         ):
1121
-            service = service_get(context, sc.service_id)
1119
+            if sc.deleted_at is not None:
1120
+                continue
1121
+
1122
+            s = service_get(context, sc.service_id)
1123
+            if service and not s.name == service:
1124
+                continue
1125
+
1126
+            if type and not sc.type == type:
1127
+                    continue
1128
+
1129
+            if component and not sc.name == component:
1130
+                continue
1131
+
1122 1132
             for sw in service_worker_get_by_host_for_service_component(
1123 1133
                 context,
1124 1134
                 service_component_id=sc.id
1125 1135
             ):
1126 1136
                 # TODO(mrkanag) Move this to db layer and query non deleted
1127
-                # if sw.deleted_at is not None:
1128
-                #     continue
1137
+                if sw.deleted_at is not None:
1138
+                    continue
1129 1139
 
1130 1140
                 sr[sw.pid] = (
1131 1141
                     dict(node=sn.name,
1132 1142
                          type=sc.type,
1133
-                         service=service.name,
1134
-                         component=sw.name,
1143
+                         service=s.name,
1144
+                         component=sc.name,
1145
+                         worker=sw.name,
1135 1146
                          status=utils.find_status(sw),
1136 1147
                          is_launcher=sw.is_launcher))
1137 1148
 
1138 1149
     return sr
1139 1150
 
1140 1151
 
1152
+def cleanup(context, service_component_id=None, dead_since=300):
1153
+    # clean up the dead service workers
1154
+    service_workers = \
1155
+        service_worker_get_all_by(
1156
+            context,
1157
+            service_component_id=service_component_id
1158
+        )
1159
+
1160
+    for srv_wkr in service_workers:
1161
+        if srv_wkr.deleted_at is not None:
1162
+            continue
1163
+
1164
+        if utils.find_status(srv_wkr, report_interval=dead_since):
1165
+            continue
1166
+        else:
1167
+            confs = config_get_by_name_for_service_worker(
1168
+                context,
1169
+                service_worker_id=srv_wkr.id
1170
+            )
1171
+
1172
+            for conf in confs:
1173
+                config_delete(context, conf.id)
1174
+
1175
+            service_worker_delete(context, srv_wkr.id)
1176
+
1177
+
1141 1178
 if __name__ == '__main__':
1142 1179
     from namos.common import config
1143 1180
 

Loading…
Cancel
Save