validate service types used for documentation jobs

We want the service type use to be consistent across all of the various
repos where we refer to it. This patch adds validation to ensure that
service types associated with jobs for repositories match the service
types assigned by the registry.

In the process, it also changes the job definitions for some of the
projects that were publishing to the "wrong" location. A separate patch
will establish redirects from the old locations to the new locations.

Some of the projects were not listed in the registry at all, so this
patch depends on other patches that add the missing entries.

The validation class defined here will eventually move to the
os-service-types library, when that repository is ready to receive code.

Change-Id: I2785ca8782b592c7af574e0b5a41407610873349
Depends-On: I27765d7760352f4c589168037ddd2e6792d1561f
Depends-On: Ie530edd8aa40e5b89f997526e68117cafa68ee9c
Signed-off-by: Doug Hellmann <doug@doughellmann.com>
This commit is contained in:
Doug Hellmann 2017-07-05 15:59:41 -04:00
parent bdb29f21f7
commit 7ab3387080
3 changed files with 130 additions and 11 deletions

View File

@ -225,7 +225,7 @@
- 'gate-telemetry-dsvm-integration-{name}-{node}':
node: ubuntu-xenial
- install-guide-jobs:
service: telemetry-alarming
service: alarm
- project:
@ -705,7 +705,7 @@
node: ubuntu-xenial
- periodic-python-jobs-with-oslo-master
- install-guide-jobs:
service: telemetry
service: meter
- project:
name: ceilometer-powervm
@ -6607,9 +6607,9 @@
envlist: genconfig
node: ubuntu-xenial
- api-ref-jobs:
service: shared-file-systems
service: shared-file-system
- install-guide-jobs:
service: shared-file-systems
service: shared-file-system
- '{pipeline}-manilaclient-dsvm-neutron-functional-{node}{suffix}':
node: ubuntu-xenial
pipeline: 'gate'
@ -8259,9 +8259,9 @@
node: ubuntu-xenial
suffix: ''
branch-override: default
# networking api-ref is maintained in neutron-lib repo.
# network api-ref is maintained in neutron-lib repo.
- api-ref-jobs:
service: networking
service: network
- project:
name: neutron-specs
@ -13426,7 +13426,7 @@
- openstack-publish-jobs
- openstack-releasenotes-jobs
- api-ref-jobs:
service: clustering
service: resource-cluster
- project:
name: senlin-dashboard
@ -13915,9 +13915,9 @@
- translation-jobs-newton
- translation-jobs-ocata
- api-ref-jobs:
service: object-storage
service: object-store
- install-guide-jobs:
service: object-storage
service: object-store
- 'gate-{name}-tox-xfs-tmp-{envlist}-{node}':
envlist:
- py27
@ -15440,9 +15440,9 @@
branch-override: default
backend: redis
- api-ref-jobs:
service: messaging
service: message
- install-guide-jobs:
service: messaging
service: message
- project:
name: zaqar-specs

View File

@ -14,9 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import io
import glob
import sys
import requests
import voluptuous as v
# The files uses YAML extensions like !include, therefore use the
@ -200,8 +202,124 @@ def _check_tox_builder(schema, entry):
return count
class ServiceTypeValidator(object):
# NOTE(dhellmann): This class will move to os-service-types when
# that repo is ready to start accepting code.
# The location of the service-types-authority data.
_URL = 'https://service-types.openstack.org/service-types.json' # noqa
def __init__(self):
# FIXME(dhellmann): Improve error handling and add caching.
self._raw = requests.get(self._URL).json()
# Store a mapping from the project name to the service
# data. Include api_reference_project names when present so
# that validation code can look up either type of project.
by_project = {}
for s in self._raw['services']:
for key in ['project', 'api_reference_project']:
name = s.get(key)
if name:
by_project[self._canonical_project_name(name)] = s
self._raw['by_project'] = by_project
def _canonical_project_name(self, name):
"Convert repo name to project name."
return name.rpartition('/')[-1]
@property
def url(self):
"The URL from which the data was retrieved."
return self._URL
@property
def version(self):
"The version of the data."
return self._raw['version']
@property
def forward(self):
"Mapping service type names to their aliases."
return copy.deepcopy(self._raw['forward'])
@property
def reverse(self):
"Mapping aliases to their service type names."
return copy.deepcopy(self._raw['reverse'])
@property
def services(self):
return copy.deepcopy(self._raw['services'])
def get_data_for_project(self, name):
"""Return the data value associated with the project.
:param name: A repository or project name in the form
``'openstack/project'`` or just ``'project'``.
:type name: str
:returns: dict
:raises: ValueError
"""
key = name.rpartition('/')[-1]
try:
return self._raw['by_project'][key]
except KeyError:
raise ValueError(
'No service_type was found for {}'.format(key),
)
# The jobs for which the service type needs to be checked
_API_JOBS = ['install-guide-jobs', 'api-guide-jobs', 'api-ref-jobs']
def validate_service_types():
print("Validating Service Types")
print("========================")
count = 0
# Load the current service-type-authority data
service_types = ServiceTypeValidator()
# Load the project job definitions
with io.open('jenkins/jobs/projects.yaml', 'r', encoding='utf-8') as f:
file_contents = local_yaml.load(f.read())
for item in file_contents:
project = item.get('project', {})
for job in project.get('jobs', []):
for api_job in _API_JOBS:
if api_job not in job:
continue
try:
proj_data = service_types.get_data_for_project(
project['name'])
except ValueError:
print('ERROR: Found service type reference "{}" for {} in {} '
'but not in authority list {}'.format(
job[api_job]['service'],
api_job,
project['name'],
service_types.url))
count +=1
else:
actual = job[api_job]['service']
expected = proj_data['service_type']
if actual != expected:
print('ERROR: Found service "{}" for {} '
'in {} but expected "{}"'.format(
job[api_job]['service'],
api_job,
project['name'],
expected))
count += 1
print('Found {} errors in service type settings '
'in jenkins/jobs/projects.yaml\n'.format(
count))
return count
def check_all():
errors = validate_jobs()
errors = errors or validate_service_types() # skip if formatting errors
errors = check_alphabetical() or errors
if errors:

View File

@ -70,6 +70,7 @@ deps =
PyYAML
voluptuous
jenkins-job-builder
requests
commands =
{toxinidir}/tools/jenkins-projects-checks.py