Build a transformed JSON document for publication

The YAML format is intended for human interaction. Scripts also want
to consume this data, so we'd like to publish a machine-friendly format
for them. That format adds a version and a sha, so that people have
a way to trace the published data back to its source, as well as forward
and reverse mappings built from the data to simplify the questions
consumers will wind up querying from the data.

Change-Id: Ia62e01ca4380df0a36daaf76fe90e5f808ac7773

.gitignore

@@ -3,3 +3,4 @@ AUTHORS
 ChangeLog
 doc/build
 *egg-info
+service-types.json*

doc/source/index.rst

@@ -3,8 +3,12 @@
 Service Data
 ============
 
+Source Data
+-----------
+
 The :download:`Service Types Authority <service-types.yaml>` information is
-kept in downloadable form in YAML.
+maintained in YAML for ease of human interactions and so that comments can
+be used if needed.
 
 .. literalinclude:: service-types.yaml
     :language: yaml
@@ -13,3 +17,34 @@ It is described by a :download:`Service Types Authority Schema <schema.json>`.
 
 .. literalinclude:: schema.json
     :language: json
+
+Publication
+-----------
+
+The information is also transformed into a JSON format and published to
+https://specs.openstack.org/service-types.json for ease of machine
+interactions. The published format is different than the source format.
+
+The published format contains five keys.
+
+version
+  An ISO Format Date Time string of the build time in UTC.
+
+sha
+  The git sha from which the file was built.
+
+service_types
+  A list of all of the official service types.
+
+forward
+  A mapping of official service type to aliases. Only contains entries for
+  services that have aliaes.
+
+reverse
+  A mapping of aliases to official service type.
+
+The published format is described by a
+:download:`Service Types Authority Published Schema <published-schema.json>`.
+
+.. literalinclude:: schema.json
+    :language: json

doc/source/published-schema.json

@@ -0,0 +1 @@
+../../published-schema.json

published-schema.json

@@ -0,0 +1,50 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "id": "https://specs.openstack.org/openstack/service-types-authority/_downloads/published-schema.json#",
+  "type": "object",
+  "required": ["services", "version", "sha", "forward", "reverse"],
+  "additionalProperties": false,
+  "properties": {
+    "version": {
+      "type": "string",
+      "description": "DateTime based version in ISO Format (https://tools.ietf.org/html/rfc3339#section-5.6",
+      "format": "date-time"
+    },
+    "sha": {
+      "type": "string",
+      "description": "sha of the git commit from which the file was generated",
+      "pattern": "^[a-f0-9]{40}"
+    },
+    "services": {
+      "type": "array",
+      "items": {
+        "$ref": "https://specs.openstack.org/openstack/service-types-authority/_downloads/schema.json#/definitions/service"
+      }
+    },
+    "forward": {
+      "type": "object",
+      "description": "Mapping of official service-type to historical aliases",
+      "patternProperties": {
+        "^([a-z][a-z-]+[a-z]+|ec2-api)$": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "Ordered list of historical aliases"
+        }
+      },
+      "additionalProperties": false
+    },
+    "reverse": {
+      "type": "object",
+      "description": "Reverse mapping of historical alias to official service-type",
+      "patternProperties": {
+        "^.*": {
+          "type": "string",
+          "pattern": "^([a-z][a-z-]+[a-z]+|ec2-api)$",
+          "description": "Official service-type"
+        }
+      }
+    }
+  }
+}

tox.ini

@@ -13,6 +13,7 @@ deps = -r{toxinidir}/requirements.txt
 deps = jsonschema
 commands =
   python validate.py
+  python transform.py -n
 
 [testenv:pep8]
 deps = hacking

transform.py

@@ -0,0 +1,78 @@
+# Copyright (c) 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import json
+import os
+import subprocess
+import sys
+
+import jsonschema
+import yaml
+
+
+class LocalResolver(jsonschema.RefResolver):
+    """Local Resolver class that uses the spec from this repo.
+
+    This repo contains the spec, and the specs are used against data in
+    gating jobs to validate consistency. However, the specs use URIs to
+    refer to each other for external consumption, which makes gating
+    changes tricky. Instead of fetching from the already published spec,
+    use the local one so that changes can be self-gating.
+    """
+
+    def resolve_remote(self, uri):
+        if uri.startswith('https://specs.openstack.org'):
+            filename = os.path.split(uri)[-1]
+            return json.load(open(filename, 'r'))
+        return super(LocalResolver, self).resolve_remote(uri)
+
+
+def main():
+    ret = 0
+    mapping = yaml.load(open('service-types.yaml', 'r'))
+
+    mapping['version'] = datetime.datetime.utcnow().isoformat()
+    mapping['sha'] = subprocess.check_output(
+        ['git', 'rev-parse', 'HEAD']).strip()
+    mapping['forward'] = {}
+    mapping['reverse'] = {}
+
+    for service in mapping['services']:
+        service_type = service['service_type']
+        if 'aliases' in service:
+            aliases = service['aliases']
+            mapping['forward'][service_type] = aliases
+            for alias in aliases:
+                mapping['reverse'][alias] = service_type
+
+    schema = json.load(open('published-schema.json', 'r'))
+    resolver = LocalResolver.from_schema(schema)
+    validator = jsonschema.Draft4Validator(schema, resolver=resolver)
+    for error in validator.iter_errors(mapping):
+        print(error.message)
+        ret = 1
+
+    if '-n' not in sys.argv:
+        json.dump(mapping, open('service-types.json', 'w'), indent=2)
+        versioned_file_name = 'service-types.json.{version}'.format(
+            version=mapping['version'])
+        json.dump(mapping, open(versioned_file_name, 'w'), indent=2)
+
+    return ret
+
+
+if __name__ == '__main__':
+    sys.exit(main())

validate.py

@@ -28,6 +28,21 @@ def main():
     for error in validator.iter_errors(data):
         print(error.message)
         ret = 1
+    service_types = []
+    aliases = []
+    for service in data['services']:
+        service_types.append(service['service_type'])
+        if "aliases" in service:
+            for alias in service['aliases']:
+                if alias in aliases:
+                    print("Alias {alias} appears twice".format(alias=alias))
+                    ret = 1
+                aliases.append(alias)
+    for alias in aliases:
+        if alias in service_types:
+            print("Alias {alias} conflicts with a service_type".format(
+                alias=alias))
+            ret = 1
     return ret