The Gatekeeper, or a project gating system
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

563 lines
21 KiB

  1. # Copyright 2017 Red Hat, Inc.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  4. # not use this file except in compliance with the License. You may obtain
  5. # a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12. # License for the specific language governing permissions and limitations
  13. # under the License.
  14. import io
  15. import logging
  16. import json
  17. import os
  18. import os.path
  19. import re
  20. import socket
  21. import tempfile
  22. import testtools
  23. import threading
  24. import time
  25. import zuul.web
  26. import zuul.lib.log_streamer
  27. import zuul.lib.fingergw
  28. import tests.base
  29. from tests.base import iterate_timeout, ZuulWebFixture
  30. from ws4py.client import WebSocketBaseClient
  31. class WSClient(WebSocketBaseClient):
  32. def __init__(self, port, build_uuid):
  33. self.port = port
  34. self.build_uuid = build_uuid
  35. self.results = ''
  36. self.event = threading.Event()
  37. uri = 'ws://[::1]:%s/api/tenant/tenant-one/console-stream' % port
  38. super(WSClient, self).__init__(uri)
  39. self.thread = threading.Thread(target=self.run)
  40. self.thread.start()
  41. def received_message(self, message):
  42. if message.is_text:
  43. self.results += message.data.decode('utf-8')
  44. def run(self):
  45. self.connect()
  46. req = {'uuid': self.build_uuid, 'logfile': None}
  47. self.send(json.dumps(req))
  48. self.event.set()
  49. super(WSClient, self).run()
  50. self.close()
  51. class TestLogStreamer(tests.base.BaseTestCase):
  52. def startStreamer(self, host, port, root=None):
  53. self.host = host
  54. if not root:
  55. root = tempfile.gettempdir()
  56. return zuul.lib.log_streamer.LogStreamer(self.host, port, root)
  57. def test_start_stop_ipv6(self):
  58. streamer = self.startStreamer('::1', 0)
  59. self.addCleanup(streamer.stop)
  60. port = streamer.server.socket.getsockname()[1]
  61. s = socket.create_connection((self.host, port))
  62. s.close()
  63. streamer.stop()
  64. with testtools.ExpectedException(ConnectionRefusedError):
  65. s = socket.create_connection((self.host, port))
  66. s.close()
  67. def test_start_stop_ipv4(self):
  68. streamer = self.startStreamer('127.0.0.1', 0)
  69. self.addCleanup(streamer.stop)
  70. port = streamer.server.socket.getsockname()[1]
  71. s = socket.create_connection((self.host, port))
  72. s.close()
  73. streamer.stop()
  74. with testtools.ExpectedException(ConnectionRefusedError):
  75. s = socket.create_connection((self.host, port))
  76. s.close()
  77. class TestStreaming(tests.base.AnsibleZuulTestCase):
  78. tenant_config_file = 'config/streamer/main.yaml'
  79. log = logging.getLogger("zuul.test_streaming")
  80. def setUp(self):
  81. super(TestStreaming, self).setUp()
  82. self.host = '::'
  83. self.streamer = None
  84. self.stop_streamer = False
  85. self.streaming_data = ''
  86. self.test_streaming_event = threading.Event()
  87. def stopStreamer(self):
  88. self.stop_streamer = True
  89. def startStreamer(self, port, build_uuid, root=None):
  90. if not root:
  91. root = tempfile.gettempdir()
  92. self.streamer = zuul.lib.log_streamer.LogStreamer(self.host,
  93. port, root)
  94. port = self.streamer.server.socket.getsockname()[1]
  95. s = socket.create_connection((self.host, port))
  96. self.addCleanup(s.close)
  97. req = '%s\r\n' % build_uuid
  98. s.sendall(req.encode('utf-8'))
  99. self.test_streaming_event.set()
  100. while not self.stop_streamer:
  101. data = s.recv(2048)
  102. if not data:
  103. break
  104. self.streaming_data += data.decode('utf-8')
  105. s.shutdown(socket.SHUT_RDWR)
  106. s.close()
  107. self.streamer.stop()
  108. def test_streaming(self):
  109. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  110. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  111. # We don't have any real synchronization for the ansible jobs, so
  112. # just wait until we get our running build.
  113. for x in iterate_timeout(30, "builds"):
  114. if len(self.builds):
  115. break
  116. build = self.builds[0]
  117. self.assertEqual(build.name, 'python27')
  118. build_dir = os.path.join(self.executor_server.jobdir_root, build.uuid)
  119. for x in iterate_timeout(30, "build dir"):
  120. if os.path.exists(build_dir):
  121. break
  122. # Need to wait to make sure that jobdir gets set
  123. for x in iterate_timeout(30, "jobdir"):
  124. if build.jobdir is not None:
  125. break
  126. build = self.builds[0]
  127. # Wait for the job to begin running and create the ansible log file.
  128. # The job waits to complete until the flag file exists, so we can
  129. # safely access the log here. We only open it (to force a file handle
  130. # to be kept open for it after the job finishes) but wait to read the
  131. # contents until the job is done.
  132. ansible_log = os.path.join(build.jobdir.log_root, 'job-output.txt')
  133. for x in iterate_timeout(30, "ansible log"):
  134. if os.path.exists(ansible_log):
  135. break
  136. logfile = open(ansible_log, 'r')
  137. self.addCleanup(logfile.close)
  138. # Create a thread to stream the log. We need this to be happening
  139. # before we create the flag file to tell the job to complete.
  140. streamer_thread = threading.Thread(
  141. target=self.startStreamer,
  142. args=(0, build.uuid, self.executor_server.jobdir_root,)
  143. )
  144. streamer_thread.start()
  145. self.addCleanup(self.stopStreamer)
  146. self.test_streaming_event.wait()
  147. # Allow the job to complete, which should close the streaming
  148. # connection (and terminate the thread) as well since the log file
  149. # gets closed/deleted.
  150. flag_file = os.path.join(build_dir, 'test_wait')
  151. open(flag_file, 'w').close()
  152. self.waitUntilSettled()
  153. streamer_thread.join()
  154. # Now that the job is finished, the log file has been closed by the
  155. # job and deleted. However, we still have a file handle to it, so we
  156. # can make sure that we read the entire contents at this point.
  157. # Compact the returned lines into a single string for easy comparison.
  158. file_contents = logfile.read()
  159. logfile.close()
  160. self.log.debug("\n\nFile contents: %s\n\n", file_contents)
  161. self.log.debug("\n\nStreamed: %s\n\n", self.streaming_data)
  162. self.assertEqual(file_contents, self.streaming_data)
  163. # Check that we logged a multiline debug message
  164. pattern = (r'^\d\d\d\d-\d\d-\d\d \d\d:\d\d\:\d\d\.\d\d\d\d\d\d \| '
  165. r'Debug Test Token String$')
  166. r = re.compile(pattern, re.MULTILINE)
  167. match = r.search(self.streaming_data)
  168. self.assertNotEqual(match, None)
  169. def runWSClient(self, port, build_uuid):
  170. client = WSClient(port, build_uuid)
  171. client.event.wait()
  172. return client
  173. def runFingerClient(self, build_uuid, gateway_address, event):
  174. # Wait until the gateway is started
  175. for x in iterate_timeout(30, "finger client to start"):
  176. try:
  177. # NOTE(Shrews): This causes the gateway to begin to handle
  178. # a request for which it never receives data, and thus
  179. # causes the getCommand() method to timeout (seen in the
  180. # test results, but is harmless).
  181. with socket.create_connection(gateway_address) as s:
  182. break
  183. except ConnectionRefusedError:
  184. pass
  185. with socket.create_connection(gateway_address) as s:
  186. msg = "%s\r\n" % build_uuid
  187. s.sendall(msg.encode('utf-8'))
  188. event.set() # notify we are connected and req sent
  189. while True:
  190. data = s.recv(1024)
  191. if not data:
  192. break
  193. self.streaming_data += data.decode('utf-8')
  194. s.shutdown(socket.SHUT_RDWR)
  195. def test_decode_boundaries(self):
  196. '''
  197. Test multi-byte characters crossing read buffer boundaries.
  198. The finger client used by ZuulWeb reads in increments of 1024 bytes.
  199. If the last byte is a multi-byte character, we end up with an error
  200. similar to:
  201. 'utf-8' codec can't decode byte 0xe2 in position 1023: \
  202. unexpected end of data
  203. By making the 1024th character in the log file a multi-byte character
  204. (here, the Euro character), we can test this.
  205. '''
  206. # Start the web server
  207. web = self.useFixture(
  208. ZuulWebFixture(self.gearman_server.port, self.changes, self.config,
  209. self.additional_event_queues, self.upstream_root,
  210. self.rpcclient, self.poller_events,
  211. self.git_url_with_auth, self.addCleanup,
  212. self.test_root))
  213. # Start the finger streamer daemon
  214. streamer = zuul.lib.log_streamer.LogStreamer(
  215. self.host, 0, self.executor_server.jobdir_root)
  216. self.addCleanup(streamer.stop)
  217. # Need to set the streaming port before submitting the job
  218. finger_port = streamer.server.socket.getsockname()[1]
  219. self.executor_server.log_streaming_port = finger_port
  220. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  221. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  222. # We don't have any real synchronization for the ansible jobs, so
  223. # just wait until we get our running build.
  224. for x in iterate_timeout(30, "builds"):
  225. if len(self.builds):
  226. break
  227. build = self.builds[0]
  228. self.assertEqual(build.name, 'python27')
  229. build_dir = os.path.join(self.executor_server.jobdir_root, build.uuid)
  230. for x in iterate_timeout(30, "build dir"):
  231. if os.path.exists(build_dir):
  232. break
  233. # Need to wait to make sure that jobdir gets set
  234. for x in iterate_timeout(30, "jobdir"):
  235. if build.jobdir is not None:
  236. break
  237. build = self.builds[0]
  238. # Wait for the job to begin running and create the ansible log file.
  239. # The job waits to complete until the flag file exists, so we can
  240. # safely access the log here. We only open it (to force a file handle
  241. # to be kept open for it after the job finishes) but wait to read the
  242. # contents until the job is done.
  243. ansible_log = os.path.join(build.jobdir.log_root, 'job-output.txt')
  244. for x in iterate_timeout(30, "ansible log"):
  245. if os.path.exists(ansible_log):
  246. break
  247. # Replace log file contents with the 1024th character being a
  248. # multi-byte character.
  249. with io.open(ansible_log, 'w', encoding='utf8') as f:
  250. f.write("a" * 1023)
  251. f.write(u"\u20AC")
  252. logfile = open(ansible_log, 'r')
  253. self.addCleanup(logfile.close)
  254. # Start a thread with the websocket client
  255. client1 = self.runWSClient(web.port, build.uuid)
  256. client1.event.wait()
  257. # Allow the job to complete
  258. flag_file = os.path.join(build_dir, 'test_wait')
  259. open(flag_file, 'w').close()
  260. # Wait for the websocket client to complete, which it should when
  261. # it's received the full log.
  262. client1.thread.join()
  263. self.waitUntilSettled()
  264. file_contents = logfile.read()
  265. logfile.close()
  266. self.log.debug("\n\nFile contents: %s\n\n", file_contents)
  267. self.log.debug("\n\nStreamed: %s\n\n", client1.results)
  268. self.assertEqual(file_contents, client1.results)
  269. def test_websocket_streaming(self):
  270. # Start the web server
  271. web = self.useFixture(
  272. ZuulWebFixture(self.gearman_server.port, self.changes, self.config,
  273. self.additional_event_queues, self.upstream_root,
  274. self.rpcclient, self.poller_events,
  275. self.git_url_with_auth, self.addCleanup,
  276. self.test_root))
  277. # Start the finger streamer daemon
  278. streamer = zuul.lib.log_streamer.LogStreamer(
  279. self.host, 0, self.executor_server.jobdir_root)
  280. self.addCleanup(streamer.stop)
  281. # Need to set the streaming port before submitting the job
  282. finger_port = streamer.server.socket.getsockname()[1]
  283. self.executor_server.log_streaming_port = finger_port
  284. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  285. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  286. # We don't have any real synchronization for the ansible jobs, so
  287. # just wait until we get our running build.
  288. for x in iterate_timeout(30, "build"):
  289. if len(self.builds):
  290. break
  291. build = self.builds[0]
  292. self.assertEqual(build.name, 'python27')
  293. build_dir = os.path.join(self.executor_server.jobdir_root, build.uuid)
  294. for x in iterate_timeout(30, "build dir"):
  295. if os.path.exists(build_dir):
  296. break
  297. # Need to wait to make sure that jobdir gets set
  298. for x in iterate_timeout(30, "jobdir"):
  299. if build.jobdir is not None:
  300. break
  301. build = self.builds[0]
  302. # Wait for the job to begin running and create the ansible log file.
  303. # The job waits to complete until the flag file exists, so we can
  304. # safely access the log here. We only open it (to force a file handle
  305. # to be kept open for it after the job finishes) but wait to read the
  306. # contents until the job is done.
  307. ansible_log = os.path.join(build.jobdir.log_root, 'job-output.txt')
  308. for x in iterate_timeout(30, "ansible log"):
  309. if os.path.exists(ansible_log):
  310. break
  311. logfile = open(ansible_log, 'r')
  312. self.addCleanup(logfile.close)
  313. # Start a thread with the websocket client
  314. client1 = self.runWSClient(web.port, build.uuid)
  315. client1.event.wait()
  316. client2 = self.runWSClient(web.port, build.uuid)
  317. client2.event.wait()
  318. # Allow the job to complete
  319. flag_file = os.path.join(build_dir, 'test_wait')
  320. open(flag_file, 'w').close()
  321. # Wait for the websocket client to complete, which it should when
  322. # it's received the full log.
  323. client1.thread.join()
  324. client2.thread.join()
  325. self.waitUntilSettled()
  326. file_contents = logfile.read()
  327. self.log.debug("\n\nFile contents: %s\n\n", file_contents)
  328. self.log.debug("\n\nStreamed: %s\n\n", client1.results)
  329. self.assertEqual(file_contents, client1.results)
  330. self.log.debug("\n\nStreamed: %s\n\n", client2.results)
  331. self.assertEqual(file_contents, client2.results)
  332. def test_websocket_hangup(self):
  333. # Start the web server
  334. web = self.useFixture(
  335. ZuulWebFixture(self.gearman_server.port, self.changes, self.config,
  336. self.additional_event_queues, self.upstream_root,
  337. self.rpcclient, self.poller_events,
  338. self.git_url_with_auth, self.addCleanup,
  339. self.test_root))
  340. # Start the finger streamer daemon
  341. streamer = zuul.lib.log_streamer.LogStreamer(
  342. self.host, 0, self.executor_server.jobdir_root)
  343. self.addCleanup(streamer.stop)
  344. # Need to set the streaming port before submitting the job
  345. finger_port = streamer.server.socket.getsockname()[1]
  346. self.executor_server.log_streaming_port = finger_port
  347. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  348. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  349. # We don't have any real synchronization for the ansible jobs, so
  350. # just wait until we get our running build.
  351. for x in iterate_timeout(30, "build"):
  352. if len(self.builds):
  353. break
  354. build = self.builds[0]
  355. self.assertEqual(build.name, 'python27')
  356. build_dir = os.path.join(self.executor_server.jobdir_root, build.uuid)
  357. for x in iterate_timeout(30, "build dir"):
  358. if os.path.exists(build_dir):
  359. break
  360. # Need to wait to make sure that jobdir gets set
  361. for x in iterate_timeout(30, "jobdir"):
  362. if build.jobdir is not None:
  363. break
  364. build = self.builds[0]
  365. # Wait for the job to begin running and create the ansible log file.
  366. # The job waits to complete until the flag file exists, so we can
  367. # safely access the log here.
  368. ansible_log = os.path.join(build.jobdir.log_root, 'job-output.txt')
  369. for x in iterate_timeout(30, "ansible log"):
  370. if os.path.exists(ansible_log):
  371. break
  372. # Start a thread with the websocket client
  373. client1 = self.runWSClient(web.port, build.uuid)
  374. client1.event.wait()
  375. # Wait until we've streamed everything so far
  376. for x in iterate_timeout(30, "streamer is caught up"):
  377. with open(ansible_log, 'r') as logfile:
  378. if client1.results == logfile.read():
  379. break
  380. # This is intensive, give it some time
  381. time.sleep(1)
  382. self.assertNotEqual(len(web.web.stream_manager.streamers.keys()), 0)
  383. # Hangup the client side
  384. client1.close(1000, 'test close')
  385. client1.thread.join()
  386. # The client should be de-registered shortly
  387. for x in iterate_timeout(30, "client cleanup"):
  388. if len(web.web.stream_manager.streamers.keys()) == 0:
  389. break
  390. # Allow the job to complete
  391. flag_file = os.path.join(build_dir, 'test_wait')
  392. open(flag_file, 'w').close()
  393. self.waitUntilSettled()
  394. def test_finger_gateway(self):
  395. # Start the finger streamer daemon
  396. streamer = zuul.lib.log_streamer.LogStreamer(
  397. self.host, 0, self.executor_server.jobdir_root)
  398. self.addCleanup(streamer.stop)
  399. finger_port = streamer.server.socket.getsockname()[1]
  400. # Need to set the streaming port before submitting the job
  401. self.executor_server.log_streaming_port = finger_port
  402. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
  403. self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
  404. # We don't have any real synchronization for the ansible jobs, so
  405. # just wait until we get our running build.
  406. for x in iterate_timeout(30, "build"):
  407. if len(self.builds):
  408. break
  409. build = self.builds[0]
  410. self.assertEqual(build.name, 'python27')
  411. build_dir = os.path.join(self.executor_server.jobdir_root, build.uuid)
  412. for x in iterate_timeout(30, "build dir"):
  413. if os.path.exists(build_dir):
  414. break
  415. # Need to wait to make sure that jobdir gets set
  416. for x in iterate_timeout(30, "jobdir"):
  417. if build.jobdir is not None:
  418. break
  419. # Wait for the job to begin running and create the ansible log file.
  420. # The job waits to complete until the flag file exists, so we can
  421. # safely access the log here. We only open it (to force a file handle
  422. # to be kept open for it after the job finishes) but wait to read the
  423. # contents until the job is done.
  424. ansible_log = os.path.join(build.jobdir.log_root, 'job-output.txt')
  425. for x in iterate_timeout(30, "ansible log"):
  426. if os.path.exists(ansible_log):
  427. break
  428. logfile = open(ansible_log, 'r')
  429. self.addCleanup(logfile.close)
  430. # Start the finger gateway daemon
  431. gateway = zuul.lib.fingergw.FingerGateway(
  432. ('127.0.0.1', self.gearman_server.port, None, None, None),
  433. (self.host, 0),
  434. user=None,
  435. command_socket=None,
  436. pid_file=None
  437. )
  438. gateway.start()
  439. self.addCleanup(gateway.stop)
  440. gateway_port = gateway.server.socket.getsockname()[1]
  441. gateway_address = (self.host, gateway_port)
  442. # Start a thread with the finger client
  443. finger_client_event = threading.Event()
  444. self.finger_client_results = ''
  445. finger_client_thread = threading.Thread(
  446. target=self.runFingerClient,
  447. args=(build.uuid, gateway_address, finger_client_event)
  448. )
  449. finger_client_thread.start()
  450. finger_client_event.wait()
  451. # Allow the job to complete
  452. flag_file = os.path.join(build_dir, 'test_wait')
  453. open(flag_file, 'w').close()
  454. # Wait for the finger client to complete, which it should when
  455. # it's received the full log.
  456. finger_client_thread.join()
  457. self.waitUntilSettled()
  458. file_contents = logfile.read()
  459. logfile.close()
  460. self.log.debug("\n\nFile contents: %s\n\n", file_contents)
  461. self.log.debug("\n\nStreamed: %s\n\n", self.streaming_data)
  462. self.assertEqual(file_contents, self.streaming_data)