fuel-qa/fuelweb_test/helpers/gerrit/gerrit_info_provider.py
Tatyana Leontovich abc8eb9179 Support openstack_tasks in mapping tests
Change-Id: I8b6a53148522e08055e61bf4746fa2ca43f3e2df
Closes-Bug: #1567372
2016-04-11 16:03:10 +00:00

247 lines
10 KiB
Python

# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import base64
from collections import namedtuple
import os
import re
# pylint: disable=redefined-builtin
from six.moves import xrange
# pylint: enable=redefined-builtin
from fuelweb_test import logger
from fuelweb_test import settings
from fuelweb_test.helpers.gerrit.gerrit_client import GerritClient
from fuelweb_test.helpers.gerrit import utils
class FuelLibraryModulesProvider(object):
PROJECT_ROOT_PATH = 'fuel-library'
MODULE_ROOT_PATH = 'deployment/puppet/'
OSNAILYFACTER_NAME = 'osnailyfacter'
OSNAILYFACTER_PATH = \
os.path.join(MODULE_ROOT_PATH, OSNAILYFACTER_NAME, 'modular/')
OSNAILYFACTER_ROLES_PATH = os.path.join(OSNAILYFACTER_PATH, 'roles/')
TASKS_YAML_PATH = os.path.join(OSNAILYFACTER_ROLES_PATH, 'tasks.yaml')
OPENSTACK_TASKS_PATH = os.path.join(MODULE_ROOT_PATH,
'openstack_tasks/manifests/')
OS_TASKS_YAML_PATH = os.path.join(MODULE_ROOT_PATH,
'openstack_tasks/tasks.yaml')
PUPPETFILE_PATH = 'deployment/Puppetfile'
def __init__(self, gerrit_review):
self.gerrit_review = gerrit_review
self.changed_modules = {}
self._files_list = set()
self.dependency_provider = DependencyProvider(self.gerrit_review)
@classmethod
def from_environment_vars(cls, endpoint='https://review.openstack.org'):
review = GerritClient(endpoint,
project=settings.GERRIT_PROJECT,
branch=settings.GERRIT_BRANCH,
change_id=settings.GERRIT_CHANGE_ID,
patchset_num=settings.GERRIT_PATCHSET_NUMBER)
return cls(review)
def get_changed_modules(self, dependency_lookup=True):
self._store_file_list()
self._find_modules_in_files()
self._find_modules_in_puppetfile_()
if dependency_lookup:
dependencies = self.dependency_provider.get_dependencies(
self.gerrit_review)
for dependency in dependencies:
self.gerrit_review.change_id = dependency.change_id
self.gerrit_review.patchset_num = str(dependency.patchset_num)
self._store_file_list()
self._find_modules_in_files()
self._find_modules_in_puppetfile_()
return self.changed_modules
def _store_file_list(self):
r = self._request_file_list()
text = r.text
files = utils.filter_response_text(text)
logger.debug('Changed files list {}'.format(files))
self._files_list.update(set(filter(lambda x: x != '/COMMIT_MSG',
utils.json_to_dict(files).keys())))
@utils.check_status_code(200)
def _request_file_list(self):
return self.gerrit_review.list_files()
def _find_modules_in_files(self):
for f in self._files_list:
if f.startswith(FuelLibraryModulesProvider.MODULE_ROOT_PATH):
split_path = f.split('/')
module = split_path[-1]
logger.debug('Process next module {}'.format(module))
self._add_module_from_files(module, split_path)
self._add_module_from_osnailyfacter(f, split_path)
self._add_module_from_openstack_tasks(f, split_path)
def _add_module_from_files(self, module, split_path):
if module != FuelLibraryModulesProvider.OSNAILYFACTER_NAME:
module_path = os.path.join(
FuelLibraryModulesProvider.PROJECT_ROOT_PATH, *split_path[:3]
)
logger.debug('Add module {0} from files by path {1}'.format(
module, module_path))
self._add_module(module, module_path)
def _add_module_from_openstack_tasks(self, filename, split_path):
if filename.startswith(
FuelLibraryModulesProvider.OPENSTACK_TASKS_PATH) \
and filename != FuelLibraryModulesProvider.OS_TASKS_YAML_PATH:
module = split_path[4]
module_path = os.path.join(
FuelLibraryModulesProvider.PROJECT_ROOT_PATH,
*split_path[0:-1])
logger.debug('Add module {0} from openstack_tasks '
'by path {1}'.format(module, module_path))
self._add_module(module, module_path)
def _add_module(self, module, module_path):
logger.debug('Changed modules are {}'.format(self.changed_modules))
if module in self.changed_modules:
logger.debug('Add module {} to changed modules'.format(module))
self.changed_modules[module].add(module_path)
else:
self.changed_modules[module] = {module_path}
logger.debug('Add module {} to changed modules'.format(module))
def _add_module_from_osnailyfacter(self, filename, split_path):
if filename.startswith(FuelLibraryModulesProvider.OSNAILYFACTER_PATH) \
and filename != FuelLibraryModulesProvider.TASKS_YAML_PATH:
module = split_path[4]
if module == 'roles':
module = 'roles/{}'.format(os.path.basename(filename))
module_path = os.path.join(
FuelLibraryModulesProvider.PROJECT_ROOT_PATH, *split_path[:5]
)
logger.debug('Add module {0} from osnailyfacter '
'by path {1}'.format(module, module_path))
self._add_module(module, module_path)
def _get_puppetfile_content_as_dict(self):
content_decoded = self._request_content(
FuelLibraryModulesProvider.PUPPETFILE_PATH
).text
content = base64.b64decode(content_decoded)
return {num: line for num, line in enumerate(content.split('\n'), 1)}
@utils.check_status_code(200)
def _request_content(self, filename):
return self.gerrit_review.get_content(filename)
def _get_puppetfile_diff_as_dict(self):
diff_raw = self._request_diff(
FuelLibraryModulesProvider.PUPPETFILE_PATH
).text
diff_filtered = utils.filter_response_text(diff_raw)
return utils.json_to_dict(diff_filtered)
@utils.check_status_code(200)
def _request_diff(self, filename):
return self.gerrit_review.get_diff(filename)
@staticmethod
def _get_lines_num_changed_from_diff(diff):
lines_changed = []
cursor = 1
for content in diff['content']:
diff_content = content.values()[0]
if 'ab' in content.keys():
cursor += len(diff_content)
if 'b' in content.keys():
lines_changed.extend(
xrange(cursor, len(diff_content) + cursor))
cursor += len(diff_content)
return lines_changed
@staticmethod
def _get_modules_line_num_changed_from_content(lines, content):
modules_lines_changed = []
for num in lines:
index = num
if content[index] == '' or content[index].startswith('#'):
continue
while not content[index].startswith('mod'):
index -= 1
modules_lines_changed.append(index)
return modules_lines_changed
def _add_modules_from_lines_changed(self, lines, content):
pattern = re.compile(r"mod '([a-z]+)',")
for num in lines:
match = pattern.match(content[num])
if match:
module = match.group(1)
self._add_module(
module,
os.path.join(
FuelLibraryModulesProvider.PROJECT_ROOT_PATH,
FuelLibraryModulesProvider.PUPPETFILE_PATH
)
)
def _find_modules_in_puppetfile_(self):
if FuelLibraryModulesProvider.PUPPETFILE_PATH in self._files_list:
content = self._get_puppetfile_content_as_dict()
diff = self._get_puppetfile_diff_as_dict()
diff_lines_changed = self._get_lines_num_changed_from_diff(diff)
mod_lines_changed = \
self._get_modules_line_num_changed_from_content(
diff_lines_changed, content)
self._add_modules_from_lines_changed(mod_lines_changed, content)
class DependencyProvider(object):
Dependency = namedtuple('Dependency', ['change_id', 'patchset_num'])
def __init__(self, review=None):
self.review = review
self.dependent_reviews = set()
@utils.check_status_code(200)
def _request_related_changes(self):
return self.review.get_related_changes()
def _get_dependencies_as_dict(self):
dependencies_raw = self._request_related_changes().text
dependencies_filtered = utils.filter_response_text(dependencies_raw)
return utils.json_to_dict(dependencies_filtered)
def _store_dependent_reviews(self, dependencies):
for dependency in dependencies['changes']:
if 'change_id' in dependency and \
'_current_revision_number' in dependency:
d = DependencyProvider.Dependency(
change_id=dependency['change_id'],
patchset_num=dependency['_current_revision_number']
)
if d.change_id != self.review.change_id:
self.dependent_reviews.add(d)
def get_dependencies(self, review=None):
if review:
self.review = review
dependencies = self._get_dependencies_as_dict()
self._store_dependent_reviews(dependencies)
return self.dependent_reviews