Added the support of Glance Artifact Repository

Murano's application packages can now be stored either in database
(using murano-api) or in Glance Artifact repository (so-called Glance
V3 api). This patch adds an experimental support of the latter
approach to murano engine.

As all the difference between these two storages is incapsulated in
the murano client, engine just needs to configure it properly by
passing an instance of glance v3 client on creation. This is
controlled by a 'packages_service' parameter of 'packages_opts'
configuration group. It is set to 'murano' by default and indicates
the usage of old, database-backed storage. If set to 'glance', the
murano client will encapsulate glance v3 connector and thus the
packages will be accessed from Glance Artifact Repository.

The settings of Glance client are also added to the configuration, as
well as a client factory to generate the client. As these settings may
now conflict with the settings "demo plugin", the appropriate
configuration section is renamed in the latter.

This patch also contains a couple of utility functions to transform
partial semver version specs into non-partial ones and - further - to
a set of Glance query parameters needed to filter the artifacts based
on that spec.

Change-Id: I690467e43b6b63850ebecef756635241e623554c
Implements-blueprint: artifact-repository-support

contrib/plugins/murano_exampleplugin/murano_exampleplugin/cfg.py

@@ -20,5 +20,5 @@ def init_config(conf):
         cfg.IntOpt('api_version', default=2),
         cfg.StrOpt('endpoint_type', default='publicURL')
     ]
-    conf.register_opts(opts, group="glance")
+    conf.register_opts(opts, group="images")
     return conf.glance

murano/common/config.py

@@ -229,7 +229,35 @@ packages_opts = [
 
     cfg.IntOpt('api_limit_max', default=100,
                help='Maximum number of packages to be returned in a single '
-                    'pagination request')
+                    'pagination request'),
+
+    cfg.StrOpt('packages_service', default='murano',
+               help=_('The service to store murano packages: murano (stands '
+                      'for legacy behavior using murano-api) or glance '
+                      '(stands for Glance V3 artifact repository)'))
+]
+
+glance_opts = [
+    cfg.StrOpt('url', help='Optional murano url in format '
+                           'like http://0.0.0.0:9292 used by Glance API'),
+
+    cfg.BoolOpt('insecure', default=False,
+                help='This option explicitly allows Murano to perform '
+                '"insecure" SSL connections and transfers with Glance API.'),
+
+    cfg.StrOpt('ca_file',
+               help='(SSL) Tells Murano to use the specified certificate file '
+               'to verify the peer running Glance API.'),
+
+    cfg.StrOpt('cert_file',
+               help='(SSL) Tells Murano to use the specified client '
+               'certificate file when communicating with Glance.'),
+
+    cfg.StrOpt('key_file', help='(SSL/SSH) Private key file name to '
+                                'communicate with Glance API.'),
+
+    cfg.StrOpt('endpoint_type', default='publicURL',
+               help='Glance endpoint type.')
 ]
 
 file_server = [
@@ -251,6 +279,7 @@ CONF.register_cli_opts(metadata_dir)
 CONF.register_opts(packages_opts, group='packages_opts')
 CONF.register_opts(stats_opts, group='stats')
 CONF.register_opts(networking_opts, group='networking')
+CONF.register_opts(glance_opts, group='glance')
 
 
 def parse_args(args=None, usage=None, default_config_files=None):

murano/dsl/helpers.py

@@ -241,16 +241,17 @@ def contextual(ctx):
 
 def parse_version_spec(version_spec):
     if isinstance(version_spec, semantic_version.Spec):
-        return version_spec
+        return normalize_version_spec(version_spec)
     if isinstance(version_spec, semantic_version.Version):
-        return semantic_version.Spec('==' + str(version_spec))
+        return normalize_version_spec(
+            semantic_version.Spec('==' + str(version_spec)))
     if not version_spec:
         version_spec = '0'
     version_spec = str(version_spec).translate(None, string.whitespace)
     if version_spec[0].isdigit():
         version_spec = '==' + str(version_spec)
     version_spec = semantic_version.Spec(version_spec)
-    return version_spec
+    return normalize_version_spec(version_spec)
 
 
 def parse_version(version):
@@ -347,3 +348,70 @@ def memoize(func):
         else:
             return cache[args]
     return wrap
+
+
+def normalize_version_spec(version_spec):
+    def coerce(v):
+        return semantic_version.Version('{0}.{1}.{2}'.format(
+            v.major, v.minor or 0, v.patch or 0
+        ))
+
+    def increment(v):
+        # NOTE(ativelkov): replace these implementations with next_minor() and
+        # next_major() calls when the semantic_version is updated in global
+        # requirements.
+        if v.minor is None:
+            return semantic_version.Version(
+                '.'.join(str(x) for x in [v.major + 1, 0, 0]))
+        else:
+            return semantic_version.Version(
+                '.'.join(str(x) for x in [v.major, v.minor + 1, 0]))
+
+    def extend(v):
+        return semantic_version.Version(str(v) + '-0')
+
+    transformations = {
+        '>': [('>=', (increment, extend))],
+        '>=': [('>=', (coerce,))],
+        '<': [('<', (coerce, extend))],
+        '<=': [('<', (increment, extend))],
+        '!=': [('>=', (increment, extend))],
+        '==': [('>=', (coerce,)), ('<', (increment, coerce, extend))]
+    }
+
+    new_parts = []
+    for item in version_spec.specs:
+        if item.kind == '*':
+            continue
+        elif item.spec.patch is not None:
+            new_parts.append(str(item))
+        else:
+            for op, funcs in transformations[item.kind]:
+                new_parts.append('{0}{1}'.format(
+                    op,
+                    reduce(lambda v, f: f(v), funcs, item.spec)
+                ))
+    if not new_parts:
+        return semantic_version.Spec('*')
+    return semantic_version.Spec(*new_parts)
+
+
+semver_to_api_map = {
+    '>': 'gt',
+    '>=': 'ge',
+    '<': 'lt',
+    '<=': 'le',
+    '!=': 'ne',
+    '==': 'eq'
+}
+
+
+def breakdown_spec_to_query(normalized_spec):
+    res = []
+    for item in normalized_spec.specs:
+        if item.kind == '*':
+            continue
+        else:
+            res.append("%s:%s" % (semver_to_api_map[item.kind],
+                                  item.spec))
+    return res

murano/engine/client_manager.py

@@ -23,7 +23,7 @@ import neutronclient.v2_0.client as nclient
 from oslo_config import cfg
 
 from murano.common import auth_utils
-
+from muranoclient.glance import client as art_client
 
 try:
     # integration with congress is optional
@@ -162,14 +162,33 @@ class ClientManager(object):
                     service_type='application_catalog',
                     endpoint_type=murano_settings.endpoint_type)
 
+            if CONF.packages_opts.packages_service == 'glance':
+                glance_settings = CONF.glance
+                glance_url = (glance_settings.url or
+                              keystone_client.service_catalog.url_for(
+                                  service_type='image',
+                                  endpoint_type=glance_settings.endpoint_type))
+
+                arts = art_client.Client(
+                    endpoint=glance_url, token=auth_token,
+                    insecure=glance_settings.insecure,
+                    key_file=glance_settings.key_file or None,
+                    ca_file=glance_settings.ca_file or None,
+                    cert_file=glance_settings.cert_file or None,
+                    type_name='murano',
+                    type_version=1)
+            else:
+                arts = None
+
             return muranoclient.Client(
                 endpoint=murano_url,
                 key_file=murano_settings.key_file or None,
-                cacert=murano_settings.cacert or None,
+                ca_file=murano_settings.cacert or None,
                 cert_file=murano_settings.cert_file or None,
                 insecure=murano_settings.insecure,
                 auth_url=keystone_client.auth_url,
-                token=auth_token)
+                token=auth_token,
+                artifacts_client=arts)
 
         return self.get_client('murano', use_trusts, factory)
 
@@ -198,3 +217,25 @@ class ClientManager(object):
                                         user_id=keystone_client.user_id)
 
         return self.get_client('mistral', use_trusts, factory)
+
+    def get_artifacts_client(self, use_trusts=True):
+        if not CONF.engine.use_trusts:
+            use_trusts = False
+
+        def factory(keystone_client, auth_token):
+            glance_settings = CONF.glance
+
+            glance_url = (glance_settings.url or
+                          keystone_client.service_catalog.url_for(
+                              service_type='image',
+                              endpoint_type=glance_settings.endpoint_type))
+
+            return art_client.Client(endpoint=glance_url, token=auth_token,
+                                     insecure=glance_settings.insecure,
+                                     key_file=glance_settings.key_file or None,
+                                     cacert=glance_settings.cacert or None,
+                                     cert_file=(glance_settings.cert_file or
+                                                None),
+                                     type_name='murano',
+                                     type_version=1)
+        return self.get_client('artifacts', use_trusts, factory)

murano/engine/package_loader.py

@@ -27,6 +27,7 @@ from oslo_log import log as logging
 from murano.common.i18n import _LE, _LI
 from murano.dsl import constants
 from murano.dsl import exceptions
+from murano.dsl import helpers
 from murano.dsl import package_loader
 from murano.engine import murano_package
 from murano.engine.system import system_objects
@@ -54,7 +55,9 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
             if version:
                 return packages[version]
 
-        filter_opts = {'class_name': class_name}
+        filter_opts = {'class_name': class_name,
+                       'version': helpers.breakdown_spec_to_query(
+                           version_spec)}
         try:
             package_definition = self._get_definition(filter_opts)
         except LookupError:
@@ -71,7 +74,9 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
             if version:
                 return packages[version]
 
-        filter_opts = {'fqn': package_name}
+        filter_opts = {'fqn': package_name,
+                       'version': helpers.breakdown_spec_to_query(
+                           version_spec)}
         try:
             package_definition = self._get_definition(filter_opts)
         except LookupError: