OpenStack Image Management (Glance)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

__init__.py 35KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951
  1. # Copyright 2011 OpenStack Foundation
  2. # All Rights Reserved.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. """
  16. Base test class for running non-stubbed tests (functional tests)
  17. The FunctionalTest class contains helper methods for starting the API
  18. and Registry server, grabbing the logs of each, cleaning up pidfiles,
  19. and spinning down the servers.
  20. """
  21. import atexit
  22. import datetime
  23. import errno
  24. import os
  25. import platform
  26. import shutil
  27. import signal
  28. import socket
  29. import sys
  30. import tempfile
  31. import time
  32. import fixtures
  33. from oslo_serialization import jsonutils
  34. # NOTE(jokke): simplified transition to py3, behaves like py2 xrange
  35. from six.moves import range
  36. import six.moves.urllib.parse as urlparse
  37. import testtools
  38. from glance.common import utils
  39. from glance.db.sqlalchemy import api as db_api
  40. from glance import tests as glance_tests
  41. from glance.tests import utils as test_utils
  42. execute, get_unused_port = test_utils.execute, test_utils.get_unused_port
  43. tracecmd_osmap = {'Linux': 'strace', 'FreeBSD': 'truss'}
  44. class Server(object):
  45. """
  46. Class used to easily manage starting and stopping
  47. a server during functional test runs.
  48. """
  49. def __init__(self, test_dir, port, sock=None):
  50. """
  51. Creates a new Server object.
  52. :param test_dir: The directory where all test stuff is kept. This is
  53. passed from the FunctionalTestCase.
  54. :param port: The port to start a server up on.
  55. """
  56. self.debug = True
  57. self.no_venv = False
  58. self.test_dir = test_dir
  59. self.bind_port = port
  60. self.conf_file_name = None
  61. self.conf_base = None
  62. self.paste_conf_base = None
  63. self.exec_env = None
  64. self.deployment_flavor = ''
  65. self.show_image_direct_url = False
  66. self.show_multiple_locations = False
  67. self.property_protection_file = ''
  68. self.enable_v1_api = True
  69. self.enable_v2_api = True
  70. self.enable_v1_registry = True
  71. self.enable_v2_registry = True
  72. self.needs_database = False
  73. self.log_file = None
  74. self.sock = sock
  75. self.fork_socket = True
  76. self.process_pid = None
  77. self.server_module = None
  78. self.stop_kill = False
  79. self.use_user_token = True
  80. self.send_identity_credentials = False
  81. def write_conf(self, **kwargs):
  82. """
  83. Writes the configuration file for the server to its intended
  84. destination. Returns the name of the configuration file and
  85. the over-ridden config content (may be useful for populating
  86. error messages).
  87. """
  88. if not self.conf_base:
  89. raise RuntimeError("Subclass did not populate config_base!")
  90. conf_override = self.__dict__.copy()
  91. if kwargs:
  92. conf_override.update(**kwargs)
  93. # A config file and paste.ini to use just for this test...we don't want
  94. # to trample on currently-running Glance servers, now do we?
  95. conf_dir = os.path.join(self.test_dir, 'etc')
  96. conf_filepath = os.path.join(conf_dir, "%s.conf" % self.server_name)
  97. if os.path.exists(conf_filepath):
  98. os.unlink(conf_filepath)
  99. paste_conf_filepath = conf_filepath.replace(".conf", "-paste.ini")
  100. if os.path.exists(paste_conf_filepath):
  101. os.unlink(paste_conf_filepath)
  102. utils.safe_mkdirs(conf_dir)
  103. def override_conf(filepath, overridden):
  104. with open(filepath, 'w') as conf_file:
  105. conf_file.write(overridden)
  106. conf_file.flush()
  107. return conf_file.name
  108. overridden_core = self.conf_base % conf_override
  109. self.conf_file_name = override_conf(conf_filepath, overridden_core)
  110. overridden_paste = ''
  111. if self.paste_conf_base:
  112. overridden_paste = self.paste_conf_base % conf_override
  113. override_conf(paste_conf_filepath, overridden_paste)
  114. overridden = ('==Core config==\n%s\n==Paste config==\n%s' %
  115. (overridden_core, overridden_paste))
  116. return self.conf_file_name, overridden
  117. def start(self, expect_exit=True, expected_exitcode=0, **kwargs):
  118. """
  119. Starts the server.
  120. Any kwargs passed to this method will override the configuration
  121. value in the conf file used in starting the servers.
  122. """
  123. # Ensure the configuration file is written
  124. self.write_conf(**kwargs)
  125. self.create_database()
  126. cmd = ("%(server_module)s --config-file %(conf_file_name)s"
  127. % {"server_module": self.server_module,
  128. "conf_file_name": self.conf_file_name})
  129. cmd = "%s -m %s" % (sys.executable, cmd)
  130. # close the sock and release the unused port closer to start time
  131. if self.exec_env:
  132. exec_env = self.exec_env.copy()
  133. else:
  134. exec_env = {}
  135. pass_fds = set()
  136. if self.sock:
  137. if not self.fork_socket:
  138. self.sock.close()
  139. self.sock = None
  140. else:
  141. fd = os.dup(self.sock.fileno())
  142. exec_env[utils.GLANCE_TEST_SOCKET_FD_STR] = str(fd)
  143. pass_fds.add(fd)
  144. self.sock.close()
  145. self.process_pid = test_utils.fork_exec(cmd,
  146. logfile=os.devnull,
  147. exec_env=exec_env,
  148. pass_fds=pass_fds)
  149. self.stop_kill = not expect_exit
  150. if self.pid_file:
  151. pf = open(self.pid_file, 'w')
  152. pf.write('%d\n' % self.process_pid)
  153. pf.close()
  154. if not expect_exit:
  155. rc = 0
  156. try:
  157. os.kill(self.process_pid, 0)
  158. except OSError:
  159. raise RuntimeError("The process did not start")
  160. else:
  161. rc = test_utils.wait_for_fork(
  162. self.process_pid,
  163. expected_exitcode=expected_exitcode)
  164. # avoid an FD leak
  165. if self.sock:
  166. os.close(fd)
  167. self.sock = None
  168. return (rc, '', '')
  169. def reload(self, expect_exit=True, expected_exitcode=0, **kwargs):
  170. """
  171. Start and stop the service to reload
  172. Any kwargs passed to this method will override the configuration
  173. value in the conf file used in starting the servers.
  174. """
  175. self.stop()
  176. return self.start(expect_exit=expect_exit,
  177. expected_exitcode=expected_exitcode, **kwargs)
  178. def create_database(self):
  179. """Create database if required for this server"""
  180. if self.needs_database:
  181. conf_dir = os.path.join(self.test_dir, 'etc')
  182. utils.safe_mkdirs(conf_dir)
  183. conf_filepath = os.path.join(conf_dir, 'glance-manage.conf')
  184. with open(conf_filepath, 'w') as conf_file:
  185. conf_file.write('[DEFAULT]\n')
  186. conf_file.write('sql_connection = %s' % self.sql_connection)
  187. conf_file.flush()
  188. glance_db_env = 'GLANCE_DB_TEST_SQLITE_FILE'
  189. if glance_db_env in os.environ:
  190. # use the empty db created and cached as a tempfile
  191. # instead of spending the time creating a new one
  192. db_location = os.environ[glance_db_env]
  193. os.system('cp %s %s/tests.sqlite'
  194. % (db_location, self.test_dir))
  195. else:
  196. cmd = ('%s -m glance.cmd.manage --config-file %s db sync' %
  197. (sys.executable, conf_filepath))
  198. execute(cmd, no_venv=self.no_venv, exec_env=self.exec_env,
  199. expect_exit=True)
  200. # copy the clean db to a temp location so that it
  201. # can be reused for future tests
  202. (osf, db_location) = tempfile.mkstemp()
  203. os.close(osf)
  204. os.system('cp %s/tests.sqlite %s'
  205. % (self.test_dir, db_location))
  206. os.environ[glance_db_env] = db_location
  207. # cleanup the temp file when the test suite is
  208. # complete
  209. def _delete_cached_db():
  210. try:
  211. os.remove(os.environ[glance_db_env])
  212. except Exception:
  213. glance_tests.logger.exception(
  214. "Error cleaning up the file %s" %
  215. os.environ[glance_db_env])
  216. atexit.register(_delete_cached_db)
  217. def stop(self):
  218. """
  219. Spin down the server.
  220. """
  221. if not self.process_pid:
  222. raise Exception('why is this being called? %s' % self.server_name)
  223. if self.stop_kill:
  224. os.kill(self.process_pid, signal.SIGTERM)
  225. rc = test_utils.wait_for_fork(self.process_pid, raise_error=False)
  226. return (rc, '', '')
  227. def dump_log(self):
  228. if not self.log_file:
  229. return "log_file not set for {name}".format(name=self.server_name)
  230. elif not os.path.exists(self.log_file):
  231. return "{log_file} for {name} did not exist".format(
  232. log_file=self.log_file, name=self.server_name)
  233. with open(self.log_file, 'r') as fptr:
  234. return fptr.read().strip()
  235. class ApiServer(Server):
  236. """
  237. Server object that starts/stops/manages the API server
  238. """
  239. def __init__(self, test_dir, port, policy_file, delayed_delete=False,
  240. pid_file=None, sock=None, **kwargs):
  241. super(ApiServer, self).__init__(test_dir, port, sock=sock)
  242. self.server_name = 'api'
  243. self.server_module = 'glance.cmd.%s' % self.server_name
  244. self.default_store = kwargs.get("default_store", "file")
  245. self.bind_host = "127.0.0.1"
  246. self.registry_host = "127.0.0.1"
  247. self.key_file = ""
  248. self.cert_file = ""
  249. self.metadata_encryption_key = "012345678901234567890123456789ab"
  250. self.image_dir = os.path.join(self.test_dir, "images")
  251. self.pid_file = pid_file or os.path.join(self.test_dir, "api.pid")
  252. self.log_file = os.path.join(self.test_dir, "api.log")
  253. self.image_size_cap = 1099511627776
  254. self.delayed_delete = delayed_delete
  255. self.owner_is_tenant = True
  256. self.workers = 0
  257. self.scrub_time = 5
  258. self.image_cache_dir = os.path.join(self.test_dir,
  259. 'cache')
  260. self.image_cache_driver = 'sqlite'
  261. self.policy_file = policy_file
  262. self.policy_default_rule = 'default'
  263. self.property_protection_rule_format = 'roles'
  264. self.image_member_quota = 10
  265. self.image_property_quota = 10
  266. self.image_tag_quota = 10
  267. self.image_location_quota = 2
  268. self.disable_path = None
  269. self.needs_database = True
  270. default_sql_connection = 'sqlite:////%s/tests.sqlite' % self.test_dir
  271. self.sql_connection = os.environ.get('GLANCE_TEST_SQL_CONNECTION',
  272. default_sql_connection)
  273. self.data_api = kwargs.get("data_api",
  274. "glance.db.sqlalchemy.api")
  275. self.user_storage_quota = '0'
  276. self.lock_path = self.test_dir
  277. self.location_strategy = 'location_order'
  278. self.store_type_location_strategy_preference = ""
  279. self.send_identity_headers = False
  280. self.conf_base = """[DEFAULT]
  281. debug = %(debug)s
  282. default_log_levels = eventlet.wsgi.server=DEBUG
  283. bind_host = %(bind_host)s
  284. bind_port = %(bind_port)s
  285. key_file = %(key_file)s
  286. cert_file = %(cert_file)s
  287. metadata_encryption_key = %(metadata_encryption_key)s
  288. registry_host = %(registry_host)s
  289. registry_port = %(registry_port)s
  290. use_user_token = %(use_user_token)s
  291. send_identity_credentials = %(send_identity_credentials)s
  292. log_file = %(log_file)s
  293. image_size_cap = %(image_size_cap)d
  294. delayed_delete = %(delayed_delete)s
  295. owner_is_tenant = %(owner_is_tenant)s
  296. workers = %(workers)s
  297. scrub_time = %(scrub_time)s
  298. send_identity_headers = %(send_identity_headers)s
  299. image_cache_dir = %(image_cache_dir)s
  300. image_cache_driver = %(image_cache_driver)s
  301. data_api = %(data_api)s
  302. sql_connection = %(sql_connection)s
  303. show_image_direct_url = %(show_image_direct_url)s
  304. show_multiple_locations = %(show_multiple_locations)s
  305. user_storage_quota = %(user_storage_quota)s
  306. enable_v1_api = %(enable_v1_api)s
  307. enable_v2_api = %(enable_v2_api)s
  308. lock_path = %(lock_path)s
  309. property_protection_file = %(property_protection_file)s
  310. property_protection_rule_format = %(property_protection_rule_format)s
  311. image_member_quota=%(image_member_quota)s
  312. image_property_quota=%(image_property_quota)s
  313. image_tag_quota=%(image_tag_quota)s
  314. image_location_quota=%(image_location_quota)s
  315. location_strategy=%(location_strategy)s
  316. allow_additional_image_properties = True
  317. [oslo_policy]
  318. policy_file = %(policy_file)s
  319. policy_default_rule = %(policy_default_rule)s
  320. [paste_deploy]
  321. flavor = %(deployment_flavor)s
  322. [store_type_location_strategy]
  323. store_type_preference = %(store_type_location_strategy_preference)s
  324. [glance_store]
  325. filesystem_store_datadir=%(image_dir)s
  326. default_store = %(default_store)s
  327. """
  328. self.paste_conf_base = """[pipeline:glance-api]
  329. pipeline =
  330. cors
  331. healthcheck
  332. versionnegotiation
  333. gzip
  334. unauthenticated-context
  335. rootapp
  336. [pipeline:glance-api-caching]
  337. pipeline = cors healthcheck versionnegotiation gzip unauthenticated-context
  338. cache rootapp
  339. [pipeline:glance-api-cachemanagement]
  340. pipeline =
  341. cors
  342. healthcheck
  343. versionnegotiation
  344. gzip
  345. unauthenticated-context
  346. cache
  347. cache_manage
  348. rootapp
  349. [pipeline:glance-api-fakeauth]
  350. pipeline = cors healthcheck versionnegotiation gzip fakeauth context rootapp
  351. [pipeline:glance-api-noauth]
  352. pipeline = cors healthcheck versionnegotiation gzip context rootapp
  353. [composite:rootapp]
  354. paste.composite_factory = glance.api:root_app_factory
  355. /: apiversions
  356. /v1: apiv1app
  357. /v2: apiv2app
  358. [app:apiversions]
  359. paste.app_factory = glance.api.versions:create_resource
  360. [app:apiv1app]
  361. paste.app_factory = glance.api.v1.router:API.factory
  362. [app:apiv2app]
  363. paste.app_factory = glance.api.v2.router:API.factory
  364. [filter:healthcheck]
  365. paste.filter_factory = oslo_middleware:Healthcheck.factory
  366. backends = disable_by_file
  367. disable_by_file_path = %(disable_path)s
  368. [filter:versionnegotiation]
  369. paste.filter_factory =
  370. glance.api.middleware.version_negotiation:VersionNegotiationFilter.factory
  371. [filter:gzip]
  372. paste.filter_factory = glance.api.middleware.gzip:GzipMiddleware.factory
  373. [filter:cache]
  374. paste.filter_factory = glance.api.middleware.cache:CacheFilter.factory
  375. [filter:cache_manage]
  376. paste.filter_factory =
  377. glance.api.middleware.cache_manage:CacheManageFilter.factory
  378. [filter:context]
  379. paste.filter_factory = glance.api.middleware.context:ContextMiddleware.factory
  380. [filter:unauthenticated-context]
  381. paste.filter_factory =
  382. glance.api.middleware.context:UnauthenticatedContextMiddleware.factory
  383. [filter:fakeauth]
  384. paste.filter_factory = glance.tests.utils:FakeAuthMiddleware.factory
  385. [filter:cors]
  386. paste.filter_factory = oslo_middleware.cors:filter_factory
  387. allowed_origin=http://valid.example.com
  388. """
  389. class RegistryServer(Server):
  390. """
  391. Server object that starts/stops/manages the Registry server
  392. """
  393. def __init__(self, test_dir, port, policy_file, sock=None):
  394. super(RegistryServer, self).__init__(test_dir, port, sock=sock)
  395. self.server_name = 'registry'
  396. self.server_module = 'glance.cmd.%s' % self.server_name
  397. self.needs_database = True
  398. default_sql_connection = 'sqlite:////%s/tests.sqlite' % self.test_dir
  399. self.sql_connection = os.environ.get('GLANCE_TEST_SQL_CONNECTION',
  400. default_sql_connection)
  401. self.bind_host = "127.0.0.1"
  402. self.pid_file = os.path.join(self.test_dir, "registry.pid")
  403. self.log_file = os.path.join(self.test_dir, "registry.log")
  404. self.owner_is_tenant = True
  405. self.workers = 0
  406. self.api_version = 1
  407. self.user_storage_quota = '0'
  408. self.metadata_encryption_key = "012345678901234567890123456789ab"
  409. self.policy_file = policy_file
  410. self.policy_default_rule = 'default'
  411. self.disable_path = None
  412. self.conf_base = """[DEFAULT]
  413. debug = %(debug)s
  414. bind_host = %(bind_host)s
  415. bind_port = %(bind_port)s
  416. log_file = %(log_file)s
  417. sql_connection = %(sql_connection)s
  418. sql_idle_timeout = 3600
  419. api_limit_max = 1000
  420. limit_param_default = 25
  421. owner_is_tenant = %(owner_is_tenant)s
  422. enable_v2_registry = %(enable_v2_registry)s
  423. workers = %(workers)s
  424. user_storage_quota = %(user_storage_quota)s
  425. metadata_encryption_key = %(metadata_encryption_key)s
  426. [oslo_policy]
  427. policy_file = %(policy_file)s
  428. policy_default_rule = %(policy_default_rule)s
  429. [paste_deploy]
  430. flavor = %(deployment_flavor)s
  431. """
  432. self.paste_conf_base = """[pipeline:glance-registry]
  433. pipeline = healthcheck unauthenticated-context registryapp
  434. [pipeline:glance-registry-fakeauth]
  435. pipeline = healthcheck fakeauth context registryapp
  436. [pipeline:glance-registry-trusted-auth]
  437. pipeline = healthcheck context registryapp
  438. [app:registryapp]
  439. paste.app_factory = glance.registry.api:API.factory
  440. [filter:healthcheck]
  441. paste.filter_factory = oslo_middleware:Healthcheck.factory
  442. backends = disable_by_file
  443. disable_by_file_path = %(disable_path)s
  444. [filter:context]
  445. paste.filter_factory = glance.api.middleware.context:ContextMiddleware.factory
  446. [filter:unauthenticated-context]
  447. paste.filter_factory =
  448. glance.api.middleware.context:UnauthenticatedContextMiddleware.factory
  449. [filter:fakeauth]
  450. paste.filter_factory = glance.tests.utils:FakeAuthMiddleware.factory
  451. """
  452. class ScrubberDaemon(Server):
  453. """
  454. Server object that starts/stops/manages the Scrubber server
  455. """
  456. def __init__(self, test_dir, policy_file, daemon=False, **kwargs):
  457. # NOTE(jkoelker): Set the port to 0 since we actually don't listen
  458. super(ScrubberDaemon, self).__init__(test_dir, 0)
  459. self.server_name = 'scrubber'
  460. self.server_module = 'glance.cmd.%s' % self.server_name
  461. self.daemon = daemon
  462. self.registry_host = "127.0.0.1"
  463. self.image_dir = os.path.join(self.test_dir, "images")
  464. self.scrub_time = 5
  465. self.pid_file = os.path.join(self.test_dir, "scrubber.pid")
  466. self.log_file = os.path.join(self.test_dir, "scrubber.log")
  467. self.metadata_encryption_key = "012345678901234567890123456789ab"
  468. self.lock_path = self.test_dir
  469. default_sql_connection = 'sqlite:////%s/tests.sqlite' % self.test_dir
  470. self.sql_connection = os.environ.get('GLANCE_TEST_SQL_CONNECTION',
  471. default_sql_connection)
  472. self.policy_file = policy_file
  473. self.policy_default_rule = 'default'
  474. self.send_identity_headers = False
  475. self.admin_role = 'admin'
  476. self.conf_base = """[DEFAULT]
  477. debug = %(debug)s
  478. log_file = %(log_file)s
  479. daemon = %(daemon)s
  480. wakeup_time = 2
  481. scrub_time = %(scrub_time)s
  482. metadata_encryption_key = %(metadata_encryption_key)s
  483. lock_path = %(lock_path)s
  484. sql_connection = %(sql_connection)s
  485. sql_idle_timeout = 3600
  486. [glance_store]
  487. filesystem_store_datadir=%(image_dir)s
  488. [oslo_policy]
  489. policy_file = %(policy_file)s
  490. policy_default_rule = %(policy_default_rule)s
  491. """
  492. def start(self, expect_exit=True, expected_exitcode=0, **kwargs):
  493. if 'daemon' in kwargs:
  494. expect_exit = False
  495. return super(ScrubberDaemon, self).start(
  496. expect_exit=expect_exit,
  497. expected_exitcode=expected_exitcode,
  498. **kwargs)
  499. class FunctionalTest(test_utils.BaseTestCase):
  500. """
  501. Base test class for any test that wants to test the actual
  502. servers and clients and not just the stubbed out interfaces
  503. """
  504. inited = False
  505. disabled = False
  506. launched_servers = []
  507. def setUp(self):
  508. super(FunctionalTest, self).setUp()
  509. self.test_dir = self.useFixture(fixtures.TempDir()).path
  510. self.api_protocol = 'http'
  511. self.api_port, api_sock = test_utils.get_unused_port_and_socket()
  512. self.registry_port, reg_sock = test_utils.get_unused_port_and_socket()
  513. # NOTE: Scrubber is enabled by default for the functional tests.
  514. # Please disbale it by explicitly setting 'self.include_scrubber' to
  515. # False in the test SetUps that do not require Scrubber to run.
  516. self.include_scrubber = True
  517. self.tracecmd = tracecmd_osmap.get(platform.system())
  518. conf_dir = os.path.join(self.test_dir, 'etc')
  519. utils.safe_mkdirs(conf_dir)
  520. self.copy_data_file('schema-image.json', conf_dir)
  521. self.copy_data_file('policy.json', conf_dir)
  522. self.copy_data_file('property-protections.conf', conf_dir)
  523. self.copy_data_file('property-protections-policies.conf', conf_dir)
  524. self.property_file_roles = os.path.join(conf_dir,
  525. 'property-protections.conf')
  526. property_policies = 'property-protections-policies.conf'
  527. self.property_file_policies = os.path.join(conf_dir,
  528. property_policies)
  529. self.policy_file = os.path.join(conf_dir, 'policy.json')
  530. self.api_server = ApiServer(self.test_dir,
  531. self.api_port,
  532. self.policy_file,
  533. sock=api_sock)
  534. self.registry_server = RegistryServer(self.test_dir,
  535. self.registry_port,
  536. self.policy_file,
  537. sock=reg_sock)
  538. self.scrubber_daemon = ScrubberDaemon(self.test_dir, self.policy_file)
  539. self.pid_files = [self.api_server.pid_file,
  540. self.registry_server.pid_file,
  541. self.scrubber_daemon.pid_file]
  542. self.files_to_destroy = []
  543. self.launched_servers = []
  544. # Keep track of servers we've logged so we don't double-log them.
  545. self._attached_server_logs = []
  546. self.addOnException(self.add_log_details_on_exception)
  547. if not self.disabled:
  548. # We destroy the test data store between each test case,
  549. # and recreate it, which ensures that we have no side-effects
  550. # from the tests
  551. self.addCleanup(
  552. self._reset_database, self.registry_server.sql_connection)
  553. self.addCleanup(
  554. self._reset_database, self.api_server.sql_connection)
  555. self.addCleanup(self.cleanup)
  556. self._reset_database(self.registry_server.sql_connection)
  557. self._reset_database(self.api_server.sql_connection)
  558. def set_policy_rules(self, rules):
  559. fap = open(self.policy_file, 'w')
  560. fap.write(jsonutils.dumps(rules))
  561. fap.close()
  562. def _reset_database(self, conn_string):
  563. conn_pieces = urlparse.urlparse(conn_string)
  564. if conn_string.startswith('sqlite'):
  565. # We leave behind the sqlite DB for failing tests to aid
  566. # in diagnosis, as the file size is relatively small and
  567. # won't interfere with subsequent tests as it's in a per-
  568. # test directory (which is blown-away if the test is green)
  569. pass
  570. elif conn_string.startswith('mysql'):
  571. # We can execute the MySQL client to destroy and re-create
  572. # the MYSQL database, which is easier and less error-prone
  573. # than using SQLAlchemy to do this via MetaData...trust me.
  574. database = conn_pieces.path.strip('/')
  575. loc_pieces = conn_pieces.netloc.split('@')
  576. host = loc_pieces[1]
  577. auth_pieces = loc_pieces[0].split(':')
  578. user = auth_pieces[0]
  579. password = ""
  580. if len(auth_pieces) > 1:
  581. if auth_pieces[1].strip():
  582. password = "-p%s" % auth_pieces[1]
  583. sql = ("drop database if exists %(database)s; "
  584. "create database %(database)s;") % {'database': database}
  585. cmd = ("mysql -u%(user)s %(password)s -h%(host)s "
  586. "-e\"%(sql)s\"") % {'user': user, 'password': password,
  587. 'host': host, 'sql': sql}
  588. exitcode, out, err = execute(cmd)
  589. self.assertEqual(0, exitcode)
  590. def cleanup(self):
  591. """
  592. Makes sure anything we created or started up in the
  593. tests are destroyed or spun down
  594. """
  595. # NOTE(jbresnah) call stop on each of the servers instead of
  596. # checking the pid file. stop() will wait until the child
  597. # server is dead. This eliminates the possibility of a race
  598. # between a child process listening on a port actually dying
  599. # and a new process being started
  600. servers = [self.api_server,
  601. self.registry_server,
  602. self.scrubber_daemon]
  603. for s in servers:
  604. try:
  605. s.stop()
  606. except Exception:
  607. pass
  608. for f in self.files_to_destroy:
  609. if os.path.exists(f):
  610. os.unlink(f)
  611. def start_server(self,
  612. server,
  613. expect_launch,
  614. expect_exit=True,
  615. expected_exitcode=0,
  616. **kwargs):
  617. """
  618. Starts a server on an unused port.
  619. Any kwargs passed to this method will override the configuration
  620. value in the conf file used in starting the server.
  621. :param server: the server to launch
  622. :param expect_launch: true iff the server is expected to
  623. successfully start
  624. :param expect_exit: true iff the launched process is expected
  625. to exit in a timely fashion
  626. :param expected_exitcode: expected exitcode from the launcher
  627. """
  628. self.cleanup()
  629. # Start up the requested server
  630. exitcode, out, err = server.start(expect_exit=expect_exit,
  631. expected_exitcode=expected_exitcode,
  632. **kwargs)
  633. if expect_exit:
  634. self.assertEqual(expected_exitcode, exitcode,
  635. "Failed to spin up the requested server. "
  636. "Got: %s" % err)
  637. self.launched_servers.append(server)
  638. launch_msg = self.wait_for_servers([server], expect_launch)
  639. self.assertTrue(launch_msg is None, launch_msg)
  640. def start_with_retry(self, server, port_name, max_retries,
  641. expect_launch=True,
  642. **kwargs):
  643. """
  644. Starts a server, with retries if the server launches but
  645. fails to start listening on the expected port.
  646. :param server: the server to launch
  647. :param port_name: the name of the port attribute
  648. :param max_retries: the maximum number of attempts
  649. :param expect_launch: true iff the server is expected to
  650. successfully start
  651. :param expect_exit: true iff the launched process is expected
  652. to exit in a timely fashion
  653. """
  654. launch_msg = None
  655. for i in range(max_retries):
  656. exitcode, out, err = server.start(expect_exit=not expect_launch,
  657. **kwargs)
  658. name = server.server_name
  659. self.assertEqual(0, exitcode,
  660. "Failed to spin up the %s server. "
  661. "Got: %s" % (name, err))
  662. launch_msg = self.wait_for_servers([server], expect_launch)
  663. if launch_msg:
  664. server.stop()
  665. server.bind_port = get_unused_port()
  666. setattr(self, port_name, server.bind_port)
  667. else:
  668. self.launched_servers.append(server)
  669. break
  670. self.assertTrue(launch_msg is None, launch_msg)
  671. def start_servers(self, **kwargs):
  672. """
  673. Starts the API and Registry servers (glance-control api start
  674. & glance-control registry start) on unused ports. glance-control
  675. should be installed into the python path
  676. Any kwargs passed to this method will override the configuration
  677. value in the conf file used in starting the servers.
  678. """
  679. self.cleanup()
  680. # Start up the API and default registry server
  681. # We start the registry server first, as the API server config
  682. # depends on the registry port - this ordering allows for
  683. # retrying the launch on a port clash
  684. self.start_with_retry(self.registry_server, 'registry_port', 3,
  685. **kwargs)
  686. kwargs['registry_port'] = self.registry_server.bind_port
  687. self.start_with_retry(self.api_server, 'api_port', 3, **kwargs)
  688. if self.include_scrubber:
  689. exitcode, out, err = self.scrubber_daemon.start(**kwargs)
  690. self.assertEqual(0, exitcode,
  691. "Failed to spin up the Scrubber daemon. "
  692. "Got: %s" % err)
  693. def ping_server(self, port):
  694. """
  695. Simple ping on the port. If responsive, return True, else
  696. return False.
  697. :note We use raw sockets, not ping here, since ping uses ICMP and
  698. has no concept of ports...
  699. """
  700. s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  701. try:
  702. s.connect(("127.0.0.1", port))
  703. return True
  704. except socket.error:
  705. return False
  706. finally:
  707. s.close()
  708. def ping_server_ipv6(self, port):
  709. """
  710. Simple ping on the port. If responsive, return True, else
  711. return False.
  712. :note We use raw sockets, not ping here, since ping uses ICMP and
  713. has no concept of ports...
  714. The function uses IPv6 (therefore AF_INET6 and ::1).
  715. """
  716. s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
  717. try:
  718. s.connect(("::1", port))
  719. return True
  720. except socket.error:
  721. return False
  722. finally:
  723. s.close()
  724. def wait_for_servers(self, servers, expect_launch=True, timeout=30):
  725. """
  726. Tight loop, waiting for the given server port(s) to be available.
  727. Returns when all are pingable. There is a timeout on waiting
  728. for the servers to come up.
  729. :param servers: Glance server ports to ping
  730. :param expect_launch: Optional, true iff the server(s) are
  731. expected to successfully start
  732. :param timeout: Optional, defaults to 30 seconds
  733. :returns: None if launch expectation is met, otherwise an
  734. assertion message
  735. """
  736. now = datetime.datetime.now()
  737. timeout_time = now + datetime.timedelta(seconds=timeout)
  738. replied = []
  739. while (timeout_time > now):
  740. pinged = 0
  741. for server in servers:
  742. if self.ping_server(server.bind_port):
  743. pinged += 1
  744. if server not in replied:
  745. replied.append(server)
  746. if pinged == len(servers):
  747. msg = 'Unexpected server launch status'
  748. return None if expect_launch else msg
  749. now = datetime.datetime.now()
  750. time.sleep(0.05)
  751. failed = list(set(servers) - set(replied))
  752. msg = 'Unexpected server launch status for: '
  753. for f in failed:
  754. msg += ('%s, ' % f.server_name)
  755. if os.path.exists(f.pid_file):
  756. pid = f.process_pid
  757. trace = f.pid_file.replace('.pid', '.trace')
  758. if self.tracecmd:
  759. cmd = '%s -p %d -o %s' % (self.tracecmd, pid, trace)
  760. try:
  761. execute(cmd, raise_error=False, expect_exit=False)
  762. except OSError as e:
  763. if e.errno == errno.ENOENT:
  764. raise RuntimeError('No executable found for "%s" '
  765. 'command.' % self.tracecmd)
  766. else:
  767. raise
  768. time.sleep(0.5)
  769. if os.path.exists(trace):
  770. msg += ('\n%s:\n%s\n' % (self.tracecmd,
  771. open(trace).read()))
  772. self.add_log_details(failed)
  773. return msg if expect_launch else None
  774. def stop_server(self, server):
  775. """
  776. Called to stop a single server in a normal fashion using the
  777. glance-control stop method to gracefully shut the server down.
  778. :param server: the server to stop
  779. """
  780. # Spin down the requested server
  781. server.stop()
  782. def stop_servers(self):
  783. """
  784. Called to stop the started servers in a normal fashion. Note
  785. that cleanup() will stop the servers using a fairly draconian
  786. method of sending a SIGTERM signal to the servers. Here, we use
  787. the glance-control stop method to gracefully shut the server down.
  788. This method also asserts that the shutdown was clean, and so it
  789. is meant to be called during a normal test case sequence.
  790. """
  791. # Spin down the API and default registry server
  792. self.stop_server(self.api_server)
  793. self.stop_server(self.registry_server)
  794. if self.include_scrubber:
  795. self.stop_server(self.scrubber_daemon)
  796. self._reset_database(self.registry_server.sql_connection)
  797. def run_sql_cmd(self, sql):
  798. """
  799. Provides a crude mechanism to run manual SQL commands for backend
  800. DB verification within the functional tests.
  801. The raw result set is returned.
  802. """
  803. engine = db_api.get_engine()
  804. return engine.execute(sql)
  805. def copy_data_file(self, file_name, dst_dir):
  806. src_file_name = os.path.join('glance/tests/etc', file_name)
  807. shutil.copy(src_file_name, dst_dir)
  808. dst_file_name = os.path.join(dst_dir, file_name)
  809. return dst_file_name
  810. def add_log_details_on_exception(self, *args, **kwargs):
  811. self.add_log_details()
  812. def add_log_details(self, servers=None):
  813. for s in servers or self.launched_servers:
  814. if s.log_file not in self._attached_server_logs:
  815. self._attached_server_logs.append(s.log_file)
  816. self.addDetail(
  817. s.server_name, testtools.content.text_content(s.dump_log()))