Fixtures for Fuel Library Noop tests framework
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

library.rb 15KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427
  1. require 'yaml'
  2. require 'set'
  3. module Noop
  4. class Manager
  5. # Recursively find file in the folder
  6. # @param root [String,Pathname]
  7. # @param exclude [Array<Pathname>]
  8. # @return [Array<Pathname>]
  9. def find_files(root, path_from=nil, exclude=[], &block)
  10. exclude = [exclude] unless exclude.is_a? Array
  11. root = Noop::Utils.convert_to_path root
  12. files = []
  13. begin
  14. root.children.each do |path|
  15. next if exclude.include? path.basename
  16. if path.file?
  17. if block_given?
  18. next unless block.call path
  19. end
  20. path = path.relative_path_from path_from if path_from
  21. files << path
  22. else
  23. files << find_files(path, path_from, exclude, &block)
  24. end
  25. end
  26. rescue
  27. []
  28. end
  29. files.flatten
  30. end
  31. # Scan the spec directory and gather the list of spec files
  32. # @return [Array<Pathname>]
  33. def spec_file_names
  34. return @spec_file_names if @spec_file_names
  35. error "No #{Noop::Config.dir_path_task_spec} directory!" unless Noop::Config.dir_path_task_spec.directory?
  36. @spec_file_names = find_files(Noop::Config.dir_path_task_spec, Noop::Config.dir_path_task_spec) do |file|
  37. file.to_s.end_with? '_spec.rb'
  38. end
  39. end
  40. # Scan the Hiera directory and gather the list of Hiera files
  41. # @return [Array<Pathname>]
  42. def hiera_file_names
  43. return @hiera_file_names if @hiera_file_names
  44. error "No #{Noop::Config.dir_path_hiera} directory!" unless Noop::Config.dir_path_hiera.directory?
  45. exclude = [ Noop::Config.dir_name_hiera_override, Noop::Config.dir_name_globals ]
  46. @hiera_file_names = find_files(Noop::Config.dir_path_hiera, Noop::Config.dir_path_hiera, exclude) do |file|
  47. file.to_s.end_with? '.yaml'
  48. end
  49. end
  50. # Scan the facts directory and gather the list of facts files
  51. # @return [Array<Pathname>]
  52. def facts_file_names
  53. return @facts_file_names if @facts_file_names
  54. error "No #{Noop::Config.dir_path_facts} directory!" unless Noop::Config.dir_path_facts.directory?
  55. exclude = [ Noop::Config.dir_name_facts_override ]
  56. @facts_file_names = find_files(Noop::Config.dir_path_facts, Noop::Config.dir_path_facts, exclude) do |file|
  57. file.to_s.end_with? '.yaml'
  58. end
  59. end
  60. # Scan the tasks directory and gather the list of task files
  61. # @return [Array<Pathname>]
  62. def task_file_names
  63. return @task_file_names if @task_file_names
  64. error "No #{Noop::Config.dir_path_tasks_local} directory!" unless Noop::Config.dir_path_tasks_local.directory?
  65. @task_file_names = find_files(Noop::Config.dir_path_tasks_local, Noop::Config.dir_path_tasks_local) do |file|
  66. file.to_s.end_with? '.pp'
  67. end
  68. end
  69. # Read the task deployment graph metadata files in the library:
  70. # Find all 'tasks.yaml' files in the puppet directory.
  71. # Read them all to a Hash by their ids.
  72. # Find all 'groups' records and resolve their 'tasks' reference
  73. # by pointing referenced tasks to this group instead.
  74. # @return [Hash<String => Hash>]
  75. def task_graph_metadata
  76. return @task_graph_metadata if @task_graph_metadata
  77. @task_graph_metadata = {}
  78. error "No #{Noop::Config.dir_path_modules_local} directory!" unless Noop::Config.dir_path_modules_local.directory?
  79. Noop::Config.dir_path_modules_local.find do |task_file|
  80. next unless task_file.file?
  81. next unless task_file.to_s.end_with? 'tasks.yaml'
  82. begin
  83. tasks = YAML.load_file task_file
  84. rescue
  85. next
  86. end
  87. tasks.each do |task|
  88. id = task['id']
  89. @task_graph_metadata[id] = task
  90. end
  91. end
  92. @task_graph_metadata.each do |id, group_task|
  93. next unless group_task['type'] == 'group' and group_task['tasks'].is_a? Array
  94. group_task['tasks'].each do |task|
  95. next unless @task_graph_metadata[task]
  96. @task_graph_metadata[task]['groups'] = [] unless @task_graph_metadata[task]['groups'].is_a? Array
  97. @task_graph_metadata[task]['groups'] << id
  98. end
  99. end
  100. @task_graph_metadata
  101. end
  102. # Try to determine the roles each spec should be run in using
  103. # the deployment graph metadata. Take a list of groups or roles
  104. # and form a set of them.
  105. # @return [Hash<Pathname => Set>]
  106. def assign_spec_to_roles
  107. return @assign_spec_to_roles if @assign_spec_to_roles
  108. @assign_spec_to_roles = {}
  109. task_graph_metadata.values.each do |task_data|
  110. roles = (task_data['groups'] or task_data['roles'] or task_data['role'])
  111. next unless roles
  112. roles = [roles] unless roles.is_a? Array
  113. file_path_manifest = task_data.fetch('parameters', {}).fetch('puppet_manifest', nil)
  114. next unless file_path_manifest
  115. file_path_manifest = Pathname.new file_path_manifest
  116. file_name_manifest = file_path_manifest.relative_path_from Noop::Config.dir_path_tasks_node
  117. file_name_spec = Noop::Utils.convert_to_spec file_name_manifest
  118. roles = Set.new roles
  119. @assign_spec_to_roles[file_name_spec] = Set.new unless @assign_spec_to_roles[file_name_spec].is_a? Set
  120. @assign_spec_to_roles[file_name_spec] += roles
  121. end
  122. @assign_spec_to_roles
  123. end
  124. # Try to determine the roles of each Hiera file.
  125. # Take 'nodes' structure and find 'node_roles' of the current node their.
  126. # Form a set of found values and add root 'role' value if found.
  127. # @return [Hash<Pathname => Set>]
  128. def assign_hiera_to_roles
  129. return @assign_hiera_to_roles if @assign_hiera_to_roles
  130. @assign_hiera_to_roles = {}
  131. hiera_file_names.each do |hiera_file|
  132. begin
  133. data = YAML.load_file(Noop::Config.dir_path_hiera + hiera_file)
  134. next unless data.is_a? Hash
  135. fqdn = data['fqdn']
  136. next unless fqdn
  137. nodes = data.fetch('network_metadata', {}).fetch('nodes', nil)
  138. next unless nodes
  139. this_node = nodes.find do |node|
  140. node.last['fqdn'] == fqdn
  141. end
  142. node_roles = this_node.last['node_roles']
  143. roles = Set.new
  144. roles.merge node_roles if node_roles.is_a? Array
  145. role = data['role']
  146. roles.add role if role
  147. @assign_hiera_to_roles[hiera_file] = roles
  148. rescue
  149. next
  150. end
  151. end
  152. @assign_hiera_to_roles
  153. end
  154. # Determine Hiera files for each spec file by calculating
  155. # the intersection between their roles sets.
  156. # If the spec file contains '*' role it should be counted
  157. # as all possible roles.
  158. # @return [Hash<Pathname => Pathname]
  159. def assign_spec_to_hiera
  160. return @assign_spec_to_hiera if @assign_spec_to_hiera
  161. @assign_spec_to_hiera = {}
  162. assign_spec_to_roles.each do |file_name_spec, spec_roles_set|
  163. hiera_files = get_hiera_for_roles spec_roles_set
  164. @assign_spec_to_hiera[file_name_spec] = hiera_files if hiera_files.any?
  165. end
  166. @assign_spec_to_hiera
  167. end
  168. # Read all spec annotations metadata.
  169. # @return [Hash<Pathname => Array>]
  170. def spec_run_metadata
  171. return @spec_run_metadata if @spec_run_metadata
  172. @spec_run_metadata = {}
  173. Noop::Config.dir_path_task_spec.find do |spec_file|
  174. next unless spec_file.file?
  175. next unless spec_file.to_s.end_with? '_spec.rb'
  176. spec_name = spec_file.relative_path_from(Noop::Config.dir_path_task_spec)
  177. spec_data = parse_spec_file spec_file
  178. @spec_run_metadata[spec_name] = spec_data if spec_data.any?
  179. end
  180. @spec_run_metadata
  181. end
  182. # Parse a spec file to find annotation entries.
  183. # @param [Pathname] task_spec
  184. # @return [Hash]
  185. def parse_spec_file(task_spec)
  186. task_spec_metadata = {}
  187. begin
  188. text = task_spec.read
  189. text.split("\n").each do |line|
  190. line = line.downcase
  191. if line =~ /^\s*#\s*(?:yamls|hiera):\s*(.*)/
  192. task_spec_metadata[:hiera] = [] unless task_spec_metadata[:hiera].is_a? Array
  193. task_spec_metadata[:hiera] += get_list_of_yamls $1
  194. end
  195. if line =~ /^\s*#\s*facts:\s*(.*)/
  196. task_spec_metadata[:facts] = [] unless task_spec_metadata[:facts].is_a? Array
  197. task_spec_metadata[:facts] += get_list_of_yamls $1
  198. end
  199. if line =~ /^\s*#\s*(?:skip_yamls|skip_hiera):\s(.*)/
  200. task_spec_metadata[:skip_hiera] = [] unless task_spec_metadata[:skip_hiera].is_a? Array
  201. task_spec_metadata[:skip_hiera] += get_list_of_yamls $1
  202. end
  203. if line =~ /^\s*#\s*skip_facts:\s(.*)/
  204. task_spec_metadata[:skip_facts] = [] unless task_spec_metadata[:skip_facts].is_a? Array
  205. task_spec_metadata[:skip_facts] += get_list_of_yamls $1
  206. end
  207. if line =~ /^\s*#\s*disable_spec/
  208. task_spec_metadata[:disable] = true
  209. end
  210. if line =~ /^\s*#\s*role:\s*(.*)/
  211. task_spec_metadata[:roles] = [] unless task_spec_metadata[:roles].is_a? Array
  212. roles = line.split /\s*,\s*|\s+/
  213. task_spec_metadata[:roles] += roles
  214. end
  215. if line =~ /^\s*#\s*run:\s*(.*)/
  216. run_record = get_list_of_yamls $1
  217. if run_record.length >= 2
  218. run_record = {
  219. :hiera => run_record[0],
  220. :facts => run_record[1],
  221. }
  222. task_spec_metadata[:runs] = [] unless task_spec_metadata[:runs].is_a? Array
  223. task_spec_metadata[:runs] << run_record
  224. end
  225. end
  226. end
  227. rescue
  228. return task_spec_metadata
  229. end
  230. task_spec_metadata
  231. end
  232. # Split a space or comma separated list of yaml files
  233. # and form an Array of the yaml file names.
  234. # @return [Array<Pathname>]
  235. def get_list_of_yamls(line)
  236. line = line.split /\s*,\s*|\s+/
  237. line.map do |yaml|
  238. yaml = Pathname.new yaml
  239. yaml = yaml.sub /$/, '.yaml' unless yaml.extname =~ /\.yaml/i
  240. yaml
  241. end
  242. end
  243. # Determine the list of run records for a spec file:
  244. # Take a list of explicitly defined runs if present.
  245. # Make product of allowed Hiera and facts yaml files to
  246. # form more run records.
  247. # Use the default facts file name if there is none
  248. # is given in the annotation.
  249. # Use the list of Hiera files determined by the intersection of
  250. # deployment graph metadata and Hiera yaml contents using roles
  251. # as a common data.
  252. def get_spec_runs(file_name_spec)
  253. file_name_spec = Noop::Utils.convert_to_path file_name_spec
  254. metadata = spec_run_metadata.fetch file_name_spec, {}
  255. metadata[:facts] = [Noop::Config.default_facts_file_name] unless metadata[:facts]
  256. if metadata[:roles]
  257. metadata[:hiera] = [] unless metadata[:hiera]
  258. metadata[:hiera] += get_hiera_for_roles metadata[:roles]
  259. end
  260. # the last default way to get hiera files list
  261. metadata[:hiera] = assign_spec_to_hiera.fetch file_name_spec, [] unless metadata[:hiera]
  262. runs = []
  263. metadata[:facts].product metadata[:hiera] do |facts, hiera|
  264. next if metadata[:skip_hiera].is_a? Array and metadata[:skip_hiera].include? hiera
  265. next if metadata[:skip_facts].is_a? Array and metadata[:skip_facts].include? hiera
  266. run_record = {
  267. :hiera => hiera,
  268. :facts => facts,
  269. }
  270. runs << run_record
  271. end
  272. runs += metadata[:runs] if metadata[:runs].is_a? Array
  273. runs
  274. end
  275. # Get a list of Hiera YAML files which roles
  276. # include the given set of roles
  277. # @param roles [Array,Set,String]
  278. # @return [Array]
  279. def get_hiera_for_roles(*roles)
  280. all_roles = Set.new
  281. roles.flatten.each do |role|
  282. if role.is_a? Set
  283. all_roles += role
  284. else
  285. all_roles.add role
  286. end
  287. end
  288. if all_roles.include? '*'
  289. assign_hiera_to_roles.keys
  290. else
  291. assign_hiera_to_roles.select do |_file_name_hiera, hiera_roles_set|
  292. roles_intersection = hiera_roles_set & all_roles
  293. roles_intersection.any?
  294. end.keys
  295. end
  296. end
  297. # Check if the given element matches this filter
  298. # @param [Array<String>]
  299. def filter_is_matched?(filter, element)
  300. return true unless filter
  301. filter = [filter] unless filter.is_a? Array
  302. filter.any? do |expression|
  303. expression = Regexp.new expression.to_s
  304. expression =~ element.to_s
  305. end
  306. end
  307. # Use filters to check if this spec file is included
  308. # @return [true,false]
  309. def spec_included?(spec)
  310. filter_is_matched? options[:filter_specs], spec
  311. end
  312. # Use filters to check if this facts file is included
  313. # @return [true,false]
  314. def facts_included?(facts)
  315. filter_is_matched? options[:filter_facts], facts
  316. end
  317. # Use filters to check if this Hiera file is included
  318. # @return [true,false]
  319. def hiera_included?(hiera)
  320. filter_is_matched? options[:filter_hiera], hiera
  321. end
  322. # Check if the globals spec should be skipped.
  323. # It should not be skipped only if it's explicitly enabled in the filter.
  324. # @return [true,false]
  325. def skip_globals?(file_name_spec)
  326. return false unless file_name_spec == Noop::Config.spec_name_globals
  327. return true unless options[:filter_specs]
  328. not spec_included? file_name_spec
  329. end
  330. # Check if the spec is disabled using the annotation
  331. # @return [true,false]
  332. def spec_is_disabled?(file_name_spec)
  333. file_name_spec = Noop::Utils.convert_to_path file_name_spec
  334. spec_run_metadata.fetch(file_name_spec, {}).fetch(:disable, false)
  335. end
  336. # Form the final list of Task objects that should be running.
  337. # Take all discovered spec files, get run records for them,
  338. # apply filters to exclude filtered records.
  339. # @return [Array<Noop::Task>]
  340. def task_list
  341. return @task_list if @task_list
  342. @task_list = []
  343. spec_file_names.each do |file_name_spec|
  344. next if spec_is_disabled? file_name_spec
  345. next if skip_globals? file_name_spec
  346. next unless spec_included? file_name_spec
  347. get_spec_runs(file_name_spec).each do |run|
  348. next unless run[:hiera] and run[:facts]
  349. next unless facts_included? run[:facts]
  350. next unless hiera_included? run[:hiera]
  351. task = Noop::Task.new file_name_spec, run[:hiera], run[:facts]
  352. task.parallel = true if parallel_run?
  353. @task_list << task
  354. end
  355. end
  356. @task_list
  357. end
  358. # Loop through all task files and find those that
  359. # do not have a corresponding spec file present
  360. # @return [Array<Pathname>]
  361. def find_tasks_without_specs
  362. task_file_names.reject do |manifest|
  363. spec = Noop::Utils.convert_to_spec manifest
  364. spec_file_names.include? spec
  365. end
  366. end
  367. # Loop through all spec files and find those that
  368. # do not have a corresponding task file present
  369. # @return [Array<Pathname>]
  370. def find_specs_without_tasks
  371. spec_file_names.reject do |spec|
  372. manifest = Noop::Utils.convert_to_manifest spec
  373. task_file_names.include? manifest
  374. end
  375. end
  376. # Loop through all spec files and find those
  377. # which have not been matched to any task
  378. # @return [Array<Pathname>]
  379. def find_unmatched_specs
  380. spec_file_names.reject do |spec|
  381. next true if spec == Noop::Config.spec_name_globals
  382. task_list.any? do |task|
  383. task.file_name_spec == spec
  384. end
  385. end
  386. end
  387. end
  388. end