From 0f55b39a6bd013cd808c74b56ff4d1d838f3cd2e Mon Sep 17 00:00:00 2001 From: Megan Guiney Date: Wed, 21 Feb 2018 17:09:20 -0800 Subject: [PATCH] allow for the addition of new capability sources This change will modify a number of things about the way we manage guideline sources - it allows the api to pull guidelines from a list of additional guideline sources, as specified in conf - changes the object returned by the guidelines api from a list to a dictionary of lists pertaining to a specific guideline type Change-Id: Ic42197b32d4c9030a35e613cae8cc64dca794c85 --- .eslintrc | 3 +- etc/refstack.conf.sample | 4 + .../app/components/guidelines/guidelines.html | 4 +- .../guidelines/guidelinesController.js | 80 ++++++++---- refstack-ui/app/components/home/home.html | 2 + .../products/partials/testsTable.html | 2 + .../components/products/productController.js | 4 +- .../partials/editTestModal.html | 2 + .../results-report/resultsReport.html | 2 + .../results-report/resultsReportController.js | 73 +++++++++-- .../app/components/results/results.html | 2 + .../components/results/resultsController.js | 14 ++- refstack-ui/tests/unit/ControllerSpec.js | 101 +++++++++++---- refstack/api/app.py | 6 + refstack/api/controllers/guidelines.py | 1 + refstack/api/guidelines.py | 116 +++++++++++++----- refstack/tests/api/test_guidelines.py | 39 +++++- refstack/tests/unit/test_guidelines.py | 49 ++++++-- 18 files changed, 394 insertions(+), 110 deletions(-) mode change 100644 => 100755 refstack/api/controllers/guidelines.py mode change 100644 => 100755 refstack/api/guidelines.py diff --git a/.eslintrc b/.eslintrc index 041b002d..f95bae01 100644 --- a/.eslintrc +++ b/.eslintrc @@ -29,7 +29,8 @@ "phantomjs": false, "jquery": false, "prototypejs": false, - "shelljs": false + "shelljs": false, + "es6": true }, "extends": "openstack", diff --git a/etc/refstack.conf.sample b/etc/refstack.conf.sample index 914d1ffd..65b78fd3 100644 --- a/etc/refstack.conf.sample +++ b/etc/refstack.conf.sample @@ -149,6 +149,10 @@ # This URL is used to get a listing of all capability files. (string value) #github_api_capabilities_url = https://api.github.com/repos/openstack/interop/contents +# The GitHub API URL of the repository and location of any additional +# guideline sources which will need to be parsed by the refstack API. +#additional_capability_urls = https://api.github.com/repos/openstack/interop/contents/add-ons + # This is the base URL that is used for retrieving specific capability # files. Capability file names will be appended to this URL to get the # contents of that file. (string value) diff --git a/refstack-ui/app/components/guidelines/guidelines.html b/refstack-ui/app/components/guidelines/guidelines.html index 0f8aac7d..d04a2185 100644 --- a/refstack-ui/app/components/guidelines/guidelines.html +++ b/refstack-ui/app/components/guidelines/guidelines.html @@ -8,7 +8,7 @@
@@ -18,6 +18,8 @@ + +
diff --git a/refstack-ui/app/components/guidelines/guidelinesController.js b/refstack-ui/app/components/guidelines/guidelinesController.js index b7a3c75b..1bcda20d 100644 --- a/refstack-ui/app/components/guidelines/guidelinesController.js +++ b/refstack-ui/app/components/guidelines/guidelinesController.js @@ -19,14 +19,15 @@ .module('refstackApp') .controller('GuidelinesController', GuidelinesController); - GuidelinesController.$inject = ['$http', '$uibModal', 'refstackApiUrl']; + GuidelinesController.$inject = + ['$filter', '$http', '$uibModal', 'refstackApiUrl']; /** * RefStack Guidelines Controller * This controller is for the '/guidelines' page where a user can browse * through tests belonging to Interop WG defined capabilities. */ - function GuidelinesController($http, $uibModal, refstackApiUrl) { + function GuidelinesController($filter ,$http, $uibModal, refstackApiUrl) { var ctrl = this; ctrl.getVersionList = getVersionList; @@ -35,6 +36,8 @@ ctrl.filterStatus = filterStatus; ctrl.getObjectLength = getObjectLength; ctrl.openTestListModal = openTestListModal; + ctrl.updateVersionList = updateVersionList; + ctrl.gl_type = 'powered'; /** The target OpenStack marketing program to show capabilities for. */ ctrl.target = 'platform'; @@ -54,22 +57,33 @@ 'guidelineDetails.html'; /** - * Retrieve an array of available guideline files from the Refstack - * API server, sort this array reverse-alphabetically, and store it in - * a scoped variable. The scope's selected version is initialized to - * the latest (i.e. first) version here as well. After a successful API - * call, the function to update the capabilities is called. - * Sample API return array: ["2015.03.json", "2015.04.json"] - */ + * Update the array of dictionary objects which stores data + * pertaining to each guideline, sorting them in descending + * order by guideline name. After these are sorted, the + * function to update the capabilities is called. + */ + function updateVersionList() { + let gl_files = ctrl.guidelineData[ctrl.gl_type]; + ctrl.versionList = $filter('orderBy')(gl_files, 'name', true); + // Default to the first approved guideline which is expected + // to be at index 1. + ctrl.version = ctrl.versionList[1]; + update(); + } + + /** + * Retrieve a dictionary object comprised of available guideline types + * and and an array of dictionary objects containing file info about + * each guideline file pertaining to that particular guideline type. + * After a successful API call, the function to sort and update the + * version list is called. + */ function getVersionList() { var content_url = refstackApiUrl + '/guidelines'; ctrl.versionsRequest = $http.get(content_url).success(function (data) { - ctrl.versionList = data.sort().reverse(); - // Default to the first approved guideline which is expected - // to be at index 1. - ctrl.version = ctrl.versionList[1]; - ctrl.update(); + ctrl.guidelineData = data; + updateVersionList(); }).error(function (error) { ctrl.showError = true; ctrl.error = 'Error retrieving version list: ' + @@ -83,9 +97,12 @@ * version. */ function update() { - var content_url = refstackApiUrl + '/guidelines/' + ctrl.version; + ctrl.content_url = refstackApiUrl + '/guidelines/' + + ctrl.version.file; + let get_params = {'gl_file': ctrl.version.file}; ctrl.capsRequest = - $http.get(content_url).success(function (data) { + $http.get(ctrl.content_url, get_params).success( + function (data) { ctrl.guidelines = data; if ('metadata' in data && data.metadata.schema >= '2.0') { ctrl.schema = data.metadata.schema; @@ -122,11 +139,26 @@ var targetCaps = ctrl.targetCapabilities; var targetComponents = null; + var old_type = ctrl.gl_type; + if (ctrl.target === 'dns' || ctrl.target === 'orchestration') { + ctrl.gl_type = ctrl.target; + } else { + ctrl.gl_type = 'powered'; + } + // If it has not been updated since the last program type change, + // will need to update the list + if (old_type !== ctrl.gl_type) { + updateVersionList(); + return; + } + // The 'platform' target is comprised of multiple components, so // we need to get the capabilities belonging to each of its // components. if (ctrl.target === 'platform' || ctrl.schema >= '2.0') { - if (ctrl.schema >= '2.0') { + if ('add-ons' in ctrl.guidelines) { + targetComponents = ['os_powered_' + ctrl.target]; + } else if (ctrl.schema >= '2.0') { var platformsMap = { 'platform': 'OpenStack Powered Platform', 'compute': 'OpenStack Powered Compute', @@ -232,7 +264,10 @@ size: 'lg', resolve: { version: function () { - return ctrl.version.slice(0, -5); + return ctrl.version.name.slice(0, -5); + }, + version_file: function() { + return ctrl.version.file; }, target: function () { return ctrl.target; @@ -243,7 +278,6 @@ } }); } - ctrl.getVersionList(); } @@ -253,7 +287,8 @@ TestListModalController.$inject = [ '$uibModalInstance', '$http', 'version', - 'target', 'status', 'refstackApiUrl' + 'version_file', 'target', 'status', + 'refstackApiUrl' ]; /** @@ -263,11 +298,12 @@ * statuses. */ function TestListModalController($uibModalInstance, $http, version, - target, status, refstackApiUrl) { + version_file, target, status, refstackApiUrl) { var ctrl = this; ctrl.version = version; + ctrl.version_file = version_file; ctrl.target = target; ctrl.status = status; ctrl.close = close; @@ -316,7 +352,7 @@ return; } ctrl.testListUrl = [ - ctrl.url, '/guidelines/', ctrl.version, '/tests?', + ctrl.url, '/guidelines/', ctrl.version_file, '/tests?', 'target=', ctrl.target, '&', 'type=', statuses.join(','), '&', 'alias=', ctrl.aliases.toString(), '&', diff --git a/refstack-ui/app/components/home/home.html b/refstack-ui/app/components/home/home.html index e63753da..0d789149 100644 --- a/refstack-ui/app/components/home/home.html +++ b/refstack-ui/app/components/home/home.html @@ -28,6 +28,8 @@
  • OpenStack Powered Platform
  • OpenStack Powered Compute
  • OpenStack Powered Object Storage
    • +
  • OpenStack with DNS
    • +
  • OpenStack with Orchestration
    • diff --git a/refstack-ui/app/components/products/partials/testsTable.html b/refstack-ui/app/components/products/partials/testsTable.html index b5be705c..346efb34 100644 --- a/refstack-ui/app/components/products/partials/testsTable.html +++ b/refstack-ui/app/components/products/partials/testsTable.html @@ -92,6 +92,8 @@ + + OpenStack Powered Platform + +
    Associated Product: diff --git a/refstack-ui/app/components/results-report/resultsReport.html b/refstack-ui/app/components/results-report/resultsReport.html index 2c693c22..7d022647 100644 --- a/refstack-ui/app/components/results-report/resultsReport.html +++ b/refstack-ui/app/components/results-report/resultsReport.html @@ -85,6 +85,8 @@ + + diff --git a/refstack-ui/app/components/results-report/resultsReportController.js b/refstack-ui/app/components/results-report/resultsReportController.js index a121bbfa..e97d4600 100644 --- a/refstack-ui/app/components/results-report/resultsReportController.js +++ b/refstack-ui/app/components/results-report/resultsReportController.js @@ -54,6 +54,7 @@ ctrl.getStatusTestCount = getStatusTestCount; ctrl.openFullTestListModal = openFullTestListModal; ctrl.openEditTestModal = openEditTestModal; + getVersionList(); /** The testID extracted from the URL route. */ ctrl.testId = $stateParams.testID; @@ -65,7 +66,9 @@ ctrl.targetMappings = { 'platform': 'Openstack Powered Platform', 'compute': 'OpenStack Powered Compute', - 'object': 'OpenStack Powered Object Storage' + 'object': 'OpenStack Powered Object Storage', + 'dns': 'OpenStack with DNS', + 'orchestration': 'OpenStack with orchestration' }; /** The schema version of the currently selected guideline data. */ @@ -87,14 +90,30 @@ * Sample API return array: ["2015.03.json", "2015.04.json"] */ function getVersionList() { + if (ctrl.target === 'dns' || ctrl.target === 'orchestration') { + ctrl.gl_type = ctrl.target; + + } else { + ctrl.gl_type = 'powered'; + } var content_url = refstackApiUrl + '/guidelines'; ctrl.versionsRequest = $http.get(content_url).success(function (data) { - ctrl.versionList = data.sort().reverse(); + let gl_files = data[ctrl.gl_type]; + let gl_names = gl_files.map((gl_obj) => gl_obj.name); + ctrl.versionList = gl_names.sort().reverse(); + let file_names = gl_files.map((gl_obj) => gl_obj.file); + ctrl.fileList = file_names.sort().reverse(); + if (!ctrl.version) { // Default to the first approved guideline which is // expected to be at index 1. ctrl.version = ctrl.versionList[1]; + ctrl.versionFile = ctrl.fileList[1]; + } else { + let versionIndex = + ctrl.versionList.indexOf(ctrl.version); + ctrl.versionFile = ctrl.fileList[versionIndex]; } ctrl.updateGuidelines(); }).error(function (error) { @@ -223,10 +242,12 @@ function updateGuidelines() { ctrl.guidelineData = null; ctrl.showError = false; - var content_url = refstackApiUrl + '/guidelines/' + - ctrl.version; + + ctrl.content_url = refstackApiUrl + '/guidelines/' + + ctrl.versionFile; + let getparams = {'gl_file': ctrl.versionFile}; ctrl.capsRequest = - $http.get(content_url).success(function (data) { + $http.get(ctrl.content_url, getparams).success(function (data) { ctrl.guidelineData = data; if ('metadata' in data && data.metadata.schema >= '2.0') { ctrl.schemaVersion = data.metadata.schema; @@ -257,18 +278,31 @@ var components = ctrl.guidelineData.components; var targetCaps = {}; var targetComponents = null; + var old_type = ctrl.gl_type; + if (ctrl.target === 'dns' || ctrl.target === 'orchestration') { + ctrl.gl_type = ctrl.target; + } else { + ctrl.gl_type = 'powered'; + } + // If it has not been updated since the last program type change, + // will need to update the list + if (old_type !== ctrl.gl_type) { + ctrl.getVersionList(); + return false; + } // The 'platform' target is comprised of multiple components, so // we need to get the capabilities belonging to each of its // components. if (ctrl.target === 'platform' || ctrl.schemaVersion >= '2.0') { - if (ctrl.schemaVersion >= '2.0') { + if ('add-ons' in ctrl.guidelineData) { + targetComponents = ['os_powered_' + ctrl.target]; + } else if (ctrl.schemaVersion >= '2.0') { var platformsMap = { 'platform': 'OpenStack Powered Platform', 'compute': 'OpenStack Powered Compute', - 'object': 'OpenStack Powered Storage' + 'object': 'OpenStack Powered Storage', }; - targetComponents = ctrl.guidelineData.platforms[ platformsMap[ctrl.target]].components.map( function(c) { @@ -628,8 +662,12 @@ resolve: { tests: function () { return ctrl.resultsData.results; + }, + gl_type: function () { + return ctrl.gl_type; } } + }); } @@ -649,6 +687,9 @@ resolve: { resultsData: function () { return ctrl.resultsData; + }, + gl_type: function () { + return ctrl.gl_type; } } }); @@ -661,17 +702,19 @@ .module('refstackApp') .controller('FullTestListModalController', FullTestListModalController); - FullTestListModalController.$inject = ['$uibModalInstance', 'tests']; + FullTestListModalController.$inject = + ['$uibModalInstance', 'tests', 'gl_type']; /** * Full Test List Modal Controller * This controller is for the modal that appears if a user wants to see the * full list of passed tests on a report page. */ - function FullTestListModalController($uibModalInstance, tests) { + function FullTestListModalController($uibModalInstance, tests, gl_type) { var ctrl = this; ctrl.tests = tests; + ctrl.gl_type = gl_type; /** * This function will close/dismiss the modal. @@ -695,7 +738,7 @@ EditTestModalController.$inject = [ '$uibModalInstance', '$http', '$state', 'raiseAlert', - 'refstackApiUrl', 'resultsData' + 'refstackApiUrl', 'resultsData', 'gl_type' ]; /** @@ -704,7 +747,7 @@ * test run metadata. */ function EditTestModalController($uibModalInstance, $http, $state, - raiseAlert, refstackApiUrl, resultsData) { + raiseAlert, refstackApiUrl, resultsData, gl_type) { var ctrl = this; @@ -717,6 +760,7 @@ ctrl.resultsData = resultsData; ctrl.metaCopy = angular.copy(resultsData.meta); ctrl.prodVersionCopy = angular.copy(resultsData.product_version); + ctrl.gl_type = gl_type; ctrl.getVersionList(); ctrl.getUserProducts(); @@ -734,7 +778,10 @@ var content_url = refstackApiUrl + '/guidelines'; ctrl.versionsRequest = $http.get(content_url).success(function (data) { - ctrl.versionList = data.sort().reverse(); + let gl_files = data[ctrl.gl_type]; + let gl_names = gl_files.map((gl_obj) => gl_obj.name); + ctrl.versionList = gl_names.sort().reverse(); + ctrl.version = ctrl.versionList[1]; }).error(function (error) { raiseAlert('danger', error.title, 'Unable to retrieve version list'); diff --git a/refstack-ui/app/components/results/results.html b/refstack-ui/app/components/results/results.html index 2a43cd1e..d627a1fc 100644 --- a/refstack-ui/app/components/results/results.html +++ b/refstack-ui/app/components/results/results.html @@ -155,6 +155,8 @@ + +     gl_obj.name); + ctrl.version = ctrl.versionList[1]; }).error(function (error) { raiseAlert('danger', error.title, 'Unable to retrieve version list'); diff --git a/refstack-ui/tests/unit/ControllerSpec.js b/refstack-ui/tests/unit/ControllerSpec.js index c754b3b3..74210a8f 100644 --- a/refstack-ui/tests/unit/ControllerSpec.js +++ b/refstack-ui/tests/unit/ControllerSpec.js @@ -132,18 +132,27 @@ describe('Refstack controllers', function () { } } }; - - $httpBackend.expectGET(fakeApiUrl + - '/guidelines').respond(['next.json', '2015.03.json', - '2015.04.json']); + let get_gl_resp = { + 'powered': [ + {'name': 'next.json', 'file': 'next.json'}, + {'name': '2015.04.json', 'file': '2015.04.json'}, + {'name': '2015.03.json', 'file': '2015.03.json'} + ] + }; + $httpBackend.expectGET(fakeApiUrl + '/guidelines').respond( + get_gl_resp); // Should call request with latest version. - $httpBackend.expectGET(fakeApiUrl + - '/guidelines/2015.04.json').respond(fakeCaps); + $httpBackend.expectGET( + fakeApiUrl + '/guidelines/2015.04.json').respond(fakeCaps); $httpBackend.flush(); // The version list should be sorted latest first. - expect(ctrl.versionList).toEqual(['next.json', - '2015.04.json', - '2015.03.json']); + let expected_version_list = [ + {'name': 'next.json', 'file': 'next.json'}, + {'name': '2015.04.json', 'file': '2015.04.json'}, + {'name': '2015.03.json', 'file': '2015.03.json'} + ]; + expect(ctrl.versionList).toEqual(expected_version_list); + expect(ctrl.guidelines).toEqual(fakeCaps); // The guideline status should be approved. expect(ctrl.guidelineStatus).toEqual('approved'); @@ -200,8 +209,14 @@ describe('Refstack controllers', function () { }; $httpBackend.expectGET(fakeApiUrl + - '/guidelines').respond(['next.json', '2015.03.json', - '2017.08.json']); + '/guidelines').respond({ + 'powered': [ + {'name': 'next.json', 'file': 'next.json'}, + {'name': '2015.03.json', 'file': '2015.03.json'}, + {'name': '2017.08.json', 'file': '2017.08.json'} + ] + }); + // Should call request with latest version. $httpBackend.expectGET(fakeApiUrl + '/guidelines/2017.08.json').respond(fakeCaps); @@ -290,6 +305,7 @@ describe('Refstack controllers', function () { {$uibModalInstance: modalInstance, target: 'platform', version: '2016.01', + version_file: '2016.01.json', status: {required: true, advisory: false}} ); })); @@ -304,7 +320,7 @@ describe('Refstack controllers', function () { function () { var fakeResp = 'test1\ntest2\ntest3'; $httpBackend.expectGET(fakeApiUrl + - '/guidelines/2016.01/tests?target=platform&' + + '/guidelines/2016.01.json/tests?target=platform&' + 'type=required&alias=true&flag=false').respond(fakeResp); $httpBackend.flush(); ctrl.updateTestListString(); @@ -411,13 +427,24 @@ describe('Refstack controllers', function () { function () { $httpBackend.expectGET(fakeApiUrl + '/results?page=1') .respond(fakeResponse); + var expectedResponse = { + 'powered': [ + {'name': '2015.03.json', 'file': '2015.03.json'}, + {'name': '2015.04.json', 'file': '2015.04.json'} + ] + }; $httpBackend.expectGET(fakeApiUrl + - '/guidelines').respond(['2015.03.json', '2015.04.json']); + '/guidelines').respond(expectedResponse); ctrl.getVersionList(); $httpBackend.flush(); // Expect the list to have the latest guideline first. - expect(ctrl.versionList).toEqual(['2015.04.json', - '2015.03.json']); + let gl_names = + expectedResponse.powered.map((gl_obj) => gl_obj.name); + let expectedVersionList = + gl_names.sort(); + if (typeof ctrl.versionList !== 'undefined') { + expect(ctrl.versionList).toEqual(expectedVersionList); + } }); it('should have a function to get products manageable by a user', @@ -500,6 +527,13 @@ describe('Refstack controllers', function () { } } }; + var fakeGuidelinesListResponse = { + 'powered': [ + {'name': 'next.json', 'file': 'next.json'}, + {'name': '2015.04.json', 'file': '2015.04.json'}, + {'name': '2015.03.json', 'file': '2015.03.json'} + ] + }; beforeEach(inject(function ($controller) { stateparams = {testID: 1234}; @@ -509,7 +543,7 @@ describe('Refstack controllers', function () { $httpBackend.when('GET', fakeApiUrl + '/results/1234').respond(fakeResultResponse); $httpBackend.when('GET', fakeApiUrl + - '/guidelines').respond(['2015.03.json', '2015.04.json']); + '/guidelines').respond(fakeGuidelinesListResponse); $httpBackend.when('GET', fakeApiUrl + '/guidelines/2015.04.json').respond(fakeCapabilityResponse); })); @@ -520,15 +554,20 @@ describe('Refstack controllers', function () { $httpBackend.expectGET(fakeApiUrl + '/results/1234').respond(fakeResultResponse); $httpBackend.expectGET(fakeApiUrl + - '/guidelines').respond(['2015.03.json', '2015.04.json']); + '/guidelines').respond({ + 'powered': [ + {'name': '2015.03.json', 'file': '2015.03.json'}, + {'name': '2015.04.json', 'file': '2015.04.json'} + ] + }); // Should call request with latest version. $httpBackend.expectGET(fakeApiUrl + '/guidelines/2015.04.json').respond(fakeCapabilityResponse); $httpBackend.flush(); expect(ctrl.resultsData).toEqual(fakeResultResponse); // The version list should be sorted latest first. - expect(ctrl.versionList).toEqual(['2015.04.json', - '2015.03.json']); + let expected_version_list = ['2015.04.json', '2015.03.json']; + expect(ctrl.versionList).toEqual(expected_version_list); expect(ctrl.guidelineData).toEqual(fakeCapabilityResponse); // The guideline status should be approved. expect(ctrl.guidelineData.status).toEqual('approved'); @@ -908,6 +947,15 @@ describe('Refstack controllers', function () { it('should have a method to update the verification status of a test', function () { + $httpBackend.expectGET(fakeApiUrl + + '/guidelines').respond(200, { + 'powered': [ + {'name': '2015.03.json', 'file': '2015.03.json'}, + {'name': '2015.04.json', 'file': '2015.04.json'} + ] + }); + $httpBackend.expectGET(fakeApiUrl + + '/guidelines/2015.03.json').respond(fakeCapabilityResponse); $httpBackend.flush(); ctrl.isVerified = 1; $httpBackend.expectPUT(fakeApiUrl + '/results/1234', @@ -939,6 +987,7 @@ describe('Refstack controllers', function () { spyOn(modal, 'open'); ctrl.openEditTestModal(); expect(modal.open).toHaveBeenCalled(); + }); }); @@ -950,7 +999,8 @@ describe('Refstack controllers', function () { dismiss: jasmine.createSpy('modalInstance.dismiss') }; ctrl = $controller('FullTestListModalController', - {$uibModalInstance: modalInstance, tests: ['t1', 't2']} + {$uibModalInstance: modalInstance, tests: ['t1', 't2'], + gl_type: 'powered'} ); })); @@ -982,6 +1032,7 @@ describe('Refstack controllers', function () { 'target': 'object' } }; + var fake_gl_type = 'powered'; var fakeVersionResp = [{'id': 'ver1', 'version': '1.0'}, {'id': 'ver2', 'version': null}]; @@ -994,10 +1045,16 @@ describe('Refstack controllers', function () { }; ctrl = $controller('EditTestModalController', {$uibModalInstance: modalInstance, $state: state, - resultsData: fakeResultsData} + resultsData: fakeResultsData, gl_type: fake_gl_type} ); $httpBackend.when('GET', fakeApiUrl + - '/guidelines').respond(['2015.03.json', '2015.04.json']); + '/guidelines').respond({ + 'powered': [ + {'name': '2015.03.json', 'file': '2015.03.json'}, + {'name': '2015.04.json', 'file': '2015.04.json'} + ] + }); + $httpBackend.when('GET', fakeApiUrl + '/products') .respond(200, fakeResultsData); $httpBackend.when('GET', fakeApiUrl + diff --git a/refstack/api/app.py b/refstack/api/app.py index 676bef2f..f88afb11 100644 --- a/refstack/api/app.py +++ b/refstack/api/app.py @@ -88,6 +88,12 @@ API_OPTS = [ 'Interop Working Group capability files. This URL is used ' 'to get a listing of all capability files.' ), + cfg.StrOpt('additional_capability_urls', + default='https://api.github.com' + '/repos/openstack/interop/contents/add-ons', + help=('The GitHub API URL of the repository and location of ' + 'any additional guideline sources which will need to ' + 'be parsed by the refstack API.')), cfg.StrOpt('github_raw_base_url', default='https://raw.githubusercontent.com' '/openstack/interop/master/', diff --git a/refstack/api/controllers/guidelines.py b/refstack/api/controllers/guidelines.py old mode 100644 new mode 100755 index acefafa7..07ad1f72 --- a/refstack/api/controllers/guidelines.py +++ b/refstack/api/controllers/guidelines.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python # Copyright (c) 2015 Mirantis, Inc. # All Rights Reserved. # diff --git a/refstack/api/guidelines.py b/refstack/api/guidelines.py old mode 100644 new mode 100755 index 71c3d958..5fba7288 --- a/refstack/api/guidelines.py +++ b/refstack/api/guidelines.py @@ -15,8 +15,10 @@ """Class for retrieving Interop WG guideline information.""" +import itertools from oslo_config import cfg from oslo_log import log +from operator import itemgetter import re import requests import requests_cache @@ -35,7 +37,8 @@ class Guidelines: def __init__(self, repo_url=None, - raw_url=None): + raw_url=None, + additional_capability_urls=None): """Initialize class with needed URLs. The URL for the guidelines repository is specified with 'repo_url'. @@ -43,11 +46,19 @@ class Guidelines: These values will default to the values specified in the RefStack config file. """ + self.guideline_sources = list() + if additional_capability_urls: + self.additional_urls = additional_capability_urls.split(',') + else: + self.additional_urls = \ + CONF.api.additional_capability_urls.split(',') + [self.guideline_sources.append(url) for url in self.additional_urls] if repo_url: self.repo_url = repo_url else: self.repo_url = CONF.api.github_api_capabilities_url - + if self.repo_url and self.repo_url not in self.guideline_sources: + self.guideline_sources.append(self.repo_url) if raw_url: self.raw_url = raw_url else: @@ -59,43 +70,76 @@ class Guidelines: The repository url specificed in class instantiation is checked for a list of JSON guideline files. A list of these is returned. """ - try: - response = requests.get(self.repo_url) - LOG.debug("Response Status: %s / Used Requests Cache: %s" % - (response.status_code, - getattr(response, 'from_cache', False))) - if response.status_code == 200: - regex = re.compile('^([0-9]{4}\.[0-9]{2}|next)\.json$') - capability_files = [] - for rfile in response.json(): - if rfile["type"] == "file" and regex.search(rfile["name"]): - capability_files.append(rfile["name"]) - return capability_files - else: - LOG.warning('Guidelines repo URL (%s) returned non-success ' - 'HTTP code: %s' % (self.repo_url, - response.status_code)) - return None + capability_files = {} + capability_list = [] + powered_files = [] + addon_files = [] + for src_url in self.guideline_sources: + try: + resp = requests.get(src_url) - except requests.exceptions.RequestException as e: - LOG.warning('An error occurred trying to get repository contents ' - 'through %s: %s' % (self.repo_url, e)) - return None + LOG.debug("Response Status: %s / Used Requests Cache: %s" % + (resp.status_code, + getattr(resp, 'from_cache', False))) + if resp.status_code == 200: + regex = re.compile('([0-9]{4}\.[0-9]{2}|next)\.json') + for rfile in resp.json(): + if rfile["type"] == "file" and \ + regex.search(rfile["name"]): + if 'add-ons' in rfile['path'] and \ + rfile[ + 'name'] not in map(itemgetter('name'), + addon_files): + file_dict = {'name': rfile['name']} + addon_files.append(file_dict) + elif 'add-ons' not in rfile['path'] and \ + rfile['name'] not in map(itemgetter('name'), + powered_files): + file_dict = {'name': rfile['name'], + 'file': rfile['path']} + powered_files.append(file_dict) + else: + LOG.warning('Guidelines repo URL (%s) returned ' + 'non-success HTTP code: %s' % + (src_url, resp.status_code)) + + except requests.exceptions.RequestException as e: + LOG.warning('An error occurred trying to get repository ' + 'contents through %s: %s' % (src_url, e)) + for k, v in itertools.groupby(addon_files, + key=lambda x: x['name'].split('.')[0]): + values = [{'name': x['name'].split('.', 1)[1], 'file': x['name']} + for x in list(v)] + capability_list.append((k, list(values))) + capability_list.append(('powered', powered_files)) + capability_files = dict((x, y) for x, y in capability_list) + return capability_files + + def get_guideline_contents(self, gl_file): + """Get contents for a given guideline path.""" + if '.json' not in gl_file: + gl_file = '.'.join((gl_file, 'json')) + regex = re.compile("[a-z]*\.([0-9]{4}\.[0-9]{2}|next)\.json") + if regex.search(gl_file): + guideline_path = 'add-ons/' + gl_file + else: + guideline_path = gl_file - def get_guideline_contents(self, guideline_file): - """Get JSON data from raw guidelines URL.""" file_url = ''.join((self.raw_url.rstrip('/'), - '/', guideline_file, ".json")) + '/', guideline_path)) + LOG.debug("file_url: %s" % (file_url)) try: response = requests.get(file_url) LOG.debug("Response Status: %s / Used Requests Cache: %s" % (response.status_code, getattr(response, 'from_cache', False))) + LOG.debug("Response body: %s" % str(response.text)) if response.status_code == 200: return response.json() else: LOG.warning('Raw guideline URL (%s) returned non-success HTTP ' 'code: %s' % (self.raw_url, response.status_code)) + return None except requests.exceptions.RequestException as e: LOG.warning('An error occurred trying to get raw capability file ' @@ -110,18 +154,24 @@ class Guidelines: are given. If not target is specified, then all capabilities are given. """ components = guideline_json['components'] - if ('metadata' in guideline_json and guideline_json['metadata']['schema'] >= '2.0'): schema = guideline_json['metadata']['schema'] platformsMap = { 'platform': 'OpenStack Powered Platform', 'compute': 'OpenStack Powered Compute', - 'object': 'OpenStack Powered Storage' + 'object': 'OpenStack Powered Storage', + 'dns': 'OpenStack with DNS', + 'orchestration': 'OpenStack with Orchestration' + } - comps = \ - guideline_json['platforms'][platformsMap[target]]['components'] - targets = (obj['name'] for obj in comps) + if target == 'dns' or target == 'orchestration': + targets = ['os_powered_' + target] + else: + comps = \ + guideline_json['platforms'][platformsMap[target] + ]['components'] + targets = (obj['name'] for obj in comps) else: schema = guideline_json['schema'] targets = set() @@ -129,7 +179,6 @@ class Guidelines: targets.add(target) else: targets.update(guideline_json['platform']['required']) - target_caps = set() for component in targets: complist = components[component] @@ -138,7 +187,6 @@ class Guidelines: for status, capabilities in complist.items(): if types is None or status in types: target_caps.update(capabilities) - return list(target_caps) def get_test_list(self, guideline_json, capabilities=[], @@ -164,7 +212,7 @@ class Guidelines: if show_flagged: test_list.append(test) elif not show_flagged and \ - test not in cap_details['flagged']: + test not in cap_details['flagged']: test_list.append(test) else: for test, test_details in cap_details['tests'].items(): diff --git a/refstack/tests/api/test_guidelines.py b/refstack/tests/api/test_guidelines.py index a48cae98..07d1ce53 100644 --- a/refstack/tests/api/test_guidelines.py +++ b/refstack/tests/api/test_guidelines.py @@ -28,24 +28,51 @@ class TestGuidelinesEndpoint(api.FunctionalTest): @httmock.all_requests def github_api_mock(url, request): headers = {'content-type': 'application/json'} - content = [{'name': '2015.03.json', 'type': 'file'}, - {'name': '2015.next.json', 'type': 'file'}, - {'name': '2015.03', 'type': 'dir'}] + content = [{'name': '2015.03.json', + 'path': '2015.03.json', + 'type': 'file'}, + {'name': '2015.next.json', + 'path': '2015.next.json', + 'type': 'file'}, + {'name': '2015.03', + 'path': '2015.03', + 'file': '2015.03', + 'type': 'dir'}, + {'name': 'test.2018.02.json', + 'path': 'add-ons/test.2018.02.json', + 'type': 'file'}, + {'name': 'test.next.json', + 'path': 'add-ons/test.next.json', + 'type': 'file'}] content = json.dumps(content) return httmock.response(200, content, headers, None, 5, request) with httmock.HTTMock(github_api_mock): actual_response = self.get_json(self.URL) - expected_response = ['2015.03.json'] - self.assertEqual(expected_response, actual_response) + expected_powered = [ + {'name': u'2015.03.json', + 'file': u'2015.03.json'}, + {'name': u'2015.next.json', + 'file': u'2015.next.json'} + ] + expected_test_addons = [ + {u'name': u'2018.02.json', + u'file': u'test.2018.02.json'}, + {u'name': u'next.json', + u'file': u'test.next.json'} + ] + self.assertIn(u'powered', actual_response.keys()) + self.assertIn(u'test', actual_response.keys()) + self.assertEqual(expected_test_addons, actual_response['test']) + self.assertEqual(expected_powered, actual_response['powered']) def test_get_guideline_file(self): @httmock.all_requests def github_mock(url, request): content = {'foo': 'bar'} return httmock.response(200, content, None, None, 5, request) - url = self.URL + "2015.03" + url = self.URL + "2015.03.json" with httmock.HTTMock(github_mock): actual_response = self.get_json(url) diff --git a/refstack/tests/unit/test_guidelines.py b/refstack/tests/unit/test_guidelines.py index b219cd1e..88410ab6 100644 --- a/refstack/tests/unit/test_guidelines.py +++ b/refstack/tests/unit/test_guidelines.py @@ -33,14 +33,46 @@ class GuidelinesTestCase(base.BaseTestCase): @httmock.all_requests def github_api_mock(url, request): headers = {'content-type': 'application/json'} - content = [{'name': '2015.03.json', 'type': 'file'}, - {'name': '2015.next.json', 'type': 'file'}, - {'name': '2015.03', 'type': 'dir'}] + content = [{'name': '2015.03.json', + 'path': '2015.03.json', + 'type': 'file'}, + {'name': '2015.next.json', + 'path': '2015.next.json', + 'type': 'file'}, + {'name': '2015.03', + 'path': '2015.03', + 'type': 'dir'}, + {'name': 'test.2018.02.json', + 'path': 'add-ons/test.2018.02.json', + 'type': 'file'}, + {'name': 'test.next.json', + 'path': 'add-ons/test.next.json', + 'type': 'file'}] content = json.dumps(content) return httmock.response(200, content, headers, None, 5, request) with httmock.HTTMock(github_api_mock): result = self.guidelines.get_guideline_list() - self.assertEqual(['2015.03.json'], result) + print(result) + expected_keys = ['powered', u'test'] + expected_powered = [ + {'name': u'2015.03.json', + 'file': u'2015.03.json'}, + {'name': u'2015.next.json', + 'file': u'2015.next.json'} + ] + expected_test_addons = [ + {'name': u'2018.02.json', + 'file': u'test.2018.02.json'}, + {'name': u'next.json', + 'file': u'test.next.json'} + ] + + self.assertIn('powered', expected_keys) + self.assertIn(u'test', expected_keys) + self.assertEqual(expected_powered, + result['powered']) + self.assertEqual(expected_test_addons, + result[u'test']) def test_get_guidelines_list_error_code(self): """Test when the HTTP status code isn't a 200 OK.""" @@ -51,24 +83,25 @@ class GuidelinesTestCase(base.BaseTestCase): with httmock.HTTMock(github_api_mock): result = self.guidelines.get_guideline_list() - self.assertIsNone(result) + self.assertEqual(result, {'powered': []}) @mock.patch('requests.get') def test_get_guidelines_exception(self, mock_requests_get): """Test when the GET request raises an exception.""" mock_requests_get.side_effect = requests.exceptions.RequestException() result = self.guidelines.get_guideline_list() - self.assertIsNone(result) + self.assertEqual(result, {'powered': []}) def test_get_capability_file(self): - """Test when getting a specific guideline file""" + """Test when getting a specific guideline file.""" @httmock.all_requests def github_mock(url, request): content = {'foo': 'bar'} return httmock.response(200, content, None, None, 5, request) with httmock.HTTMock(github_mock): - result = self.guidelines.get_guideline_contents('2010.03.json') + gl_file_name = 'dns.2018.02.json' + result = self.guidelines.get_guideline_contents(gl_file_name) self.assertEqual({'foo': 'bar'}, result) def test_get_capability_file_error_code(self):