The Gatekeeper, or a project gating system
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

server.py 6.6KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173
  1. # Copyright 2014 OpenStack Foundation
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  4. # not use this file except in compliance with the License. You may obtain
  5. # a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12. # License for the specific language governing permissions and limitations
  13. # under the License.
  14. import json
  15. import logging
  16. import threading
  17. import traceback
  18. import gear
  19. from zuul.lib import commandsocket
  20. from zuul.lib.config import get_default
  21. from zuul.merger import merger
  22. COMMANDS = ['stop']
  23. class MergeServer(object):
  24. log = logging.getLogger("zuul.MergeServer")
  25. def __init__(self, config, connections={}):
  26. self.config = config
  27. merge_root = get_default(self.config, 'merger', 'git_dir',
  28. '/var/lib/zuul/merger-git')
  29. merge_email = get_default(self.config, 'merger', 'git_user_email')
  30. merge_name = get_default(self.config, 'merger', 'git_user_name')
  31. speed_limit = get_default(
  32. config, 'merger', 'git_http_low_speed_limit', '1000')
  33. speed_time = get_default(
  34. config, 'merger', 'git_http_low_speed_time', '30')
  35. self.merger = merger.Merger(
  36. merge_root, connections, merge_email, merge_name, speed_limit,
  37. speed_time)
  38. self.command_map = dict(
  39. stop=self.stop)
  40. command_socket = get_default(
  41. self.config, 'merger', 'command_socket',
  42. '/var/lib/zuul/merger.socket')
  43. self.command_socket = commandsocket.CommandSocket(command_socket)
  44. def start(self):
  45. self._running = True
  46. self._command_running = True
  47. server = self.config.get('gearman', 'server')
  48. port = get_default(self.config, 'gearman', 'port', 4730)
  49. ssl_key = get_default(self.config, 'gearman', 'ssl_key')
  50. ssl_cert = get_default(self.config, 'gearman', 'ssl_cert')
  51. ssl_ca = get_default(self.config, 'gearman', 'ssl_ca')
  52. self.worker = gear.TextWorker('Zuul Merger')
  53. self.worker.addServer(server, port, ssl_key, ssl_cert, ssl_ca)
  54. self.log.debug("Waiting for server")
  55. self.worker.waitForServer()
  56. self.log.debug("Registering")
  57. self.register()
  58. self.log.debug("Starting command processor")
  59. self.command_socket.start()
  60. self.command_thread = threading.Thread(
  61. target=self.runCommand, name='command')
  62. self.command_thread.daemon = True
  63. self.command_thread.start()
  64. self.log.debug("Starting worker")
  65. self.thread = threading.Thread(target=self.run)
  66. self.thread.daemon = True
  67. self.thread.start()
  68. def register(self):
  69. self.worker.registerFunction("merger:merge")
  70. self.worker.registerFunction("merger:cat")
  71. self.worker.registerFunction("merger:refstate")
  72. self.worker.registerFunction("merger:fileschanges")
  73. def stop(self):
  74. self.log.debug("Stopping")
  75. self._running = False
  76. self._command_running = False
  77. self.command_socket.stop()
  78. self.worker.shutdown()
  79. self.log.debug("Stopped")
  80. def join(self):
  81. self.thread.join()
  82. def runCommand(self):
  83. while self._command_running:
  84. try:
  85. command = self.command_socket.get().decode('utf8')
  86. if command != '_stop':
  87. self.command_map[command]()
  88. except Exception:
  89. self.log.exception("Exception while processing command")
  90. def run(self):
  91. self.log.debug("Starting merge listener")
  92. while self._running:
  93. try:
  94. job = self.worker.getJob()
  95. try:
  96. if job.name == 'merger:merge':
  97. self.log.debug("Got merge job: %s" % job.unique)
  98. self.merge(job)
  99. elif job.name == 'merger:cat':
  100. self.log.debug("Got cat job: %s" % job.unique)
  101. self.cat(job)
  102. elif job.name == 'merger:refstate':
  103. self.log.debug("Got refstate job: %s" % job.unique)
  104. self.refstate(job)
  105. elif job.name == 'merger:fileschanges':
  106. self.log.debug("Got fileschanges job: %s" % job.unique)
  107. self.fileschanges(job)
  108. else:
  109. self.log.error("Unable to handle job %s" % job.name)
  110. job.sendWorkFail()
  111. except Exception:
  112. self.log.exception("Exception while running job")
  113. job.sendWorkException(traceback.format_exc())
  114. except gear.InterruptedError:
  115. return
  116. except Exception:
  117. self.log.exception("Exception while getting job")
  118. def merge(self, job):
  119. args = json.loads(job.arguments)
  120. ret = self.merger.mergeChanges(
  121. args['items'], args.get('files'),
  122. args.get('dirs'), args.get('repo_state'))
  123. result = dict(merged=(ret is not None))
  124. if ret is None:
  125. result['commit'] = result['files'] = result['repo_state'] = None
  126. else:
  127. (result['commit'], result['files'], result['repo_state'],
  128. recent, orig_commit) = ret
  129. job.sendWorkComplete(json.dumps(result))
  130. def refstate(self, job):
  131. args = json.loads(job.arguments)
  132. success, repo_state = self.merger.getRepoState(args['items'])
  133. result = dict(updated=success,
  134. repo_state=repo_state)
  135. job.sendWorkComplete(json.dumps(result))
  136. def cat(self, job):
  137. args = json.loads(job.arguments)
  138. self.merger.updateRepo(args['connection'], args['project'])
  139. files = self.merger.getFiles(args['connection'], args['project'],
  140. args['branch'], args['files'],
  141. args.get('dirs'))
  142. result = dict(updated=True,
  143. files=files)
  144. job.sendWorkComplete(json.dumps(result))
  145. def fileschanges(self, job):
  146. args = json.loads(job.arguments)
  147. self.merger.updateRepo(args['connection'], args['project'])
  148. files = self.merger.getFilesChanges(
  149. args['connection'], args['project'], args['branch'], args['tosha'])
  150. result = dict(updated=True,
  151. files=files)
  152. job.sendWorkComplete(json.dumps(result))