Initial implementation of artifacts client in tempest plugin

- Add artifacts tempest client
- Extend test utils with common utils of muranoclient to parse packages

Change-Id: I55c02a922b80f792afbe9f7be309c66115ebd6b8
targets: bp murano-glare-devstack-testing

murano_tempest_tests/config.py

@@ -35,7 +35,10 @@ application_catalog_group = cfg.OptGroup(name="application_catalog",
                                          title="Application Catalog Options")
 
 service_broker_group = cfg.OptGroup(name="service_broker",
-                                         title="Service Broker Options")
+                                    title="Service Broker Options")
+
+artifacts_group = cfg.OptGroup(name="artifacts",
+                               title="Glance Artifacts Options")
 
 ApplicationCatalogGroup = [
     # Application catalog tempest configuration
@@ -97,3 +100,25 @@ ServiceBrokerGroup = [
 
 
 ]
+
+ArtifactsGroup = [
+    # Glance artifacts options
+    cfg.StrOpt("catalog_type",
+               default="artifact",
+               help="Catalog type of Artifacts API"),
+
+    cfg.StrOpt("endpoint_type",
+               default="publicURL",
+               choices=["publicURL", "adminURL", "internalURL"],
+               help="The endpoint type for artifacts service"),
+
+    cfg.IntOpt("build_interval",
+               default=3,
+               help="Time in seconds between artifacts"
+                    " availability checks."),
+
+    cfg.IntOpt("build_timeout",
+               default=500,
+               help="Timeout in seconds to wait for a artifacts"
+                    " to become available.")
+]

murano_tempest_tests/plugin.py

@@ -39,6 +39,9 @@ class MuranoTempestPlugin(plugins.TempestPlugin):
         config.register_opt_group(
             conf, config_application_catalog.service_broker_group,
             config_application_catalog.ServiceBrokerGroup)
+        config.register_opt_group(
+            conf, config_application_catalog.artifacts_group,
+            config_application_catalog.ArtifactsGroup)
 
     def get_opt_lists(self):
         return [(config_application_catalog.application_catalog_group.name,
@@ -46,4 +49,6 @@ class MuranoTempestPlugin(plugins.TempestPlugin):
                 (config_application_catalog.service_broker_group.name,
                  config_application_catalog.ServiceBrokerGroup),
                 ('service_available',
-                 config_application_catalog.ServiceAvailableGroup)]
+                 config_application_catalog.ServiceAvailableGroup),
+                (config_application_catalog.artifacts_group.name,
+                 config_application_catalog.ArtifactsGroup)]

murano_tempest_tests/services/artifacts/artifacts_client.py

@@ -0,0 +1,102 @@
+# Copyright (c) 2016 Mirantis, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+
+from tempest import config
+from tempest.lib.common import rest_client
+
+CONF = config.CONF
+
+
+class ArtifactsClient(rest_client.RestClient):
+    """Tempest REST client for Glance Artifacts"""
+
+    def __init__(self, auth_provider):
+        super(ArtifactsClient, self).__init__(
+            auth_provider,
+            CONF.artifacts.catalog_type,
+            CONF.identity.region,
+            endpoint_type=CONF.artifacts.endpoint_type)
+        self.build_interval = CONF.artifacts.build_interval
+        self.build_timeout = CONF.artifacts.build_timeout
+
+# -----------------------------Artifacts methods-------------------------------
+
+    def list_artifacts(self):
+        uri = 'v0.1/artifacts/murano/v1'
+        resp, body = self.get(uri)
+        self.expected_success(200, resp.status)
+        return self._parse_resp(body)
+
+    def list_drafts(self):
+        uri = 'v0.1/artifacts/murano/v1/creating'
+        resp, body = self.get(uri)
+        self.expected_success(200, resp.status)
+        return self._parse_resp(body)
+
+    def list_deactivated_drafts(self):
+        uri = 'v0.1/artifacts/murano/v1/deactivated'
+        resp, body = self.get(uri)
+        self.expected_success(200, resp.status)
+        return self._parse_resp(body)
+
+    def create_artifact_draft(self, name, version, **kwargs):
+        uri = 'v0.1/artifacts/murano/v1/creating'
+        kwargs.update({'name': name, 'version': version})
+        resp, body = self.post(uri, body=json.dumps(kwargs))
+        self.expected_success(201, resp.status)
+        return self._parse_resp(body)
+
+    def publish_artifact(self, artifact_id):
+        uri = 'v0.1/artifacts/murano/v1/{0}/publish'.format(artifact_id)
+        resp, body = self.post(uri)
+        self.expected_success(200, resp.status)
+        return self._parse_resp(body)
+
+    def get_artifact(self, artifact_id):
+        uri = 'v0.1/artifacts/murano/v1/{0}'.format(artifact_id)
+        resp, body = self.get(uri)
+        self.expected_success(200, resp.status)
+        return self._parse_resp(body)
+
+    def update_artifact(self, artifact_id, body):
+        headers = {
+            'Content-Type': 'application/openstack-images-v2.1-json-patch'}
+        uri = 'v0.1/artifacts/murano/v1/{0}'.format(artifact_id)
+        resp, body = self.patch(uri, json.dumps(body), headers=headers)
+        self.expected_success(200, resp.status)
+        return self._parse_resp(body)
+
+    def delete_artifact(self, artifact_id):
+        uri = 'v0.1/artifacts/murano/v1/{0}'.format(artifact_id)
+        resp, body = self.delete(uri)
+        self.expected_success(200, resp.status)
+        return self._parse_resp(body)
+
+    def upload_blob(self, artifact_id, blob_type, data):
+        headers = {'Content-Type': 'application/octet-stream'}
+        uri = 'v0.1/artifacts/murano/v1/{0}/{1}'.format(
+            artifact_id, blob_type)
+        resp, body = self.put(uri, data, headers=headers)
+        self.expected_success(201, resp.status)
+        return self._parse_resp(body)
+
+    def download_blob(self, artifact_id, blob_type):
+        uri = 'v0.1/artifacts/murano/v1/{0}/{1}/download'.format(
+            artifact_id, blob_type)
+        resp, body = self.get(uri)
+        self.expected_success(200, resp.status)
+        return self._parse_resp(body)

murano_tempest_tests/utils.py

@@ -12,11 +12,20 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import collections
 import os
+import shutil
+import tempfile
 import uuid
 import yaml
 import zipfile
 
+from oslo_log import log as logging
+import requests
+import six
+from six.moves import urllib
+
+LOG = logging.getLogger(__name__)
 
 MANIFEST = {'Format': 'MuranoPL/1.0',
             'Type': 'Application',
@@ -113,3 +122,356 @@ def acquire_package_directory():
     expected_package_dir = '/extras/MockApp'
     app_dir = top_plugin_dir + expected_package_dir
     return app_dir
+
+
+def to_url(filename, base_url, version='', path='/', extension=''):
+    if urllib.parse.urlparse(filename).scheme in ('http', 'https'):
+        return filename
+    if not base_url:
+        raise ValueError("No base_url for repository supplied")
+    if '/' in filename or filename in ('.', '..'):
+        raise ValueError("Invalid filename path supplied: {0}".format(
+            filename))
+    version = '.' + version if version else ''
+    return urllib.parse.urljoin(base_url,
+                                path + filename + version + extension)
+
+# ----------------------Murano client common functions-------------------------
+
+
+class NoCloseProxy(object):
+    """A proxy object, that does nothing on close."""
+    def __init__(self, obj):
+        self.obj = obj
+
+    def close(self):
+        return
+
+    def __getattr__(self, name):
+        return getattr(self.obj, name)
+
+
+class File(object):
+    def __init__(self, name, binary=True):
+        self.name = name
+        self.binary = binary
+
+    def open(self):
+        mode = 'rb' if self.binary else 'r'
+        if hasattr(self.name, 'read'):
+            # NOTE(kzaitsev) We do not want to close a file object
+            # passed to File wrapper. The caller should be responsible
+            # for closing it
+            return NoCloseProxy(self.name)
+        else:
+            if os.path.isfile(self.name):
+                return open(self.name, mode)
+            url = urllib.parse.urlparse(self.name)
+            if url.scheme in ('http', 'https'):
+                resp = requests.get(self.name, stream=True)
+                if not resp.ok:
+                    raise ValueError("Got non-ok status({0}) "
+                                     "while connecting to {1}".format(
+                                         resp.status_code, self.name))
+                temp_file = tempfile.NamedTemporaryFile(mode='w+b')
+                for chunk in resp.iter_content(1024 * 1024):
+                    temp_file.write(chunk)
+                temp_file.flush()
+                return open(temp_file.name, mode)
+            raise ValueError("Can't open {0}".format(self.name))
+
+
+class FileWrapperMixin(object):
+    def __init__(self, file_wrapper):
+        self.file_wrapper = file_wrapper
+        try:
+            self._file = self.file_wrapper.open()
+        except Exception:
+            # NOTE(kzaitsev): We need to have _file available at __del__ time.
+            self._file = None
+            raise
+
+    def file(self):
+        self._file.seek(0)
+        return self._file
+
+    def close(self):
+        if self._file and not self._file.closed:
+            self._file.close()
+
+    def save(self, dst, binary=True):
+        file_name = self.file_wrapper.name
+
+        if urllib.parse.urlparse(file_name).scheme:
+            file_name = file_name.split('/')[-1]
+
+        dst = os.path.join(dst, file_name)
+
+        mode = 'wb' if binary else 'w'
+        with open(dst, mode) as dst_file:
+            self._file.seek(0)
+            shutil.copyfileobj(self._file, dst_file)
+
+    def __del__(self):
+        self.close()
+
+
+class Package(FileWrapperMixin):
+    """Represents murano package contents."""
+
+    @staticmethod
+    def from_file(file_obj):
+        if not isinstance(file_obj, File):
+            file_obj = File(file_obj)
+        return Package(file_obj)
+
+    @staticmethod
+    def from_location(name, base_url='', version='', url='', path=None):
+        """Open file using one of three possible options
+
+        If path is supplied search for name file in the path, otherwise
+        if url is supplied - open that url and finally search murano
+        repository for the package.
+        """
+        if path:
+            pkg_name = os.path.join(path, name)
+            file_name = None
+            for f in [pkg_name, pkg_name + '.zip']:
+                if os.path.exists(f):
+                    file_name = f
+            if file_name:
+                return Package.from_file(file_name)
+            LOG.error("Couldn't find file for package {0}, tried {1}".format(
+                name, [pkg_name, pkg_name + '.zip']))
+        if url:
+            return Package.from_file(url)
+        return Package.from_file(to_url(
+            name,
+            base_url=base_url,
+            version=version,
+            path='apps/',
+            extension='.zip')
+        )
+
+    @property
+    def contents(self):
+        """Contents of a package."""
+        if not hasattr(self, '_contents'):
+            try:
+                self._file.seek(0)
+                self._zip_obj = zipfile.ZipFile(
+                    six.BytesIO(self._file.read()))
+            except Exception as e:
+                LOG.error("Error {0} occurred,"
+                          " while parsing the package".format(e))
+                raise
+        return self._zip_obj
+
+    @property
+    def manifest(self):
+        """Parsed manifest file of a package."""
+        if not hasattr(self, '_manifest'):
+            try:
+                self._manifest = yaml.safe_load(
+                    self.contents.open('manifest.yaml'))
+            except Exception as e:
+                LOG.error("Error {0} occurred, while extracting "
+                          "manifest from package".format(e))
+                raise
+        return self._manifest
+
+    def images(self):
+        """Returns a list of required image specifications."""
+        if 'images.lst' not in self.contents.namelist():
+            return []
+        try:
+            return yaml.safe_load(
+                self.contents.open('images.lst')).get('Images', [])
+        except Exception:
+            return []
+
+    @property
+    def classes(self):
+        if not hasattr(self, '_classes'):
+            self._classes = {}
+            for class_name, class_file in six.iteritems(
+                    self.manifest.get('Classes', {})):
+                filename = "Classes/%s" % class_file
+                if filename not in self.contents.namelist():
+                    continue
+                klass = yaml.safe_load(self.contents.open(filename))
+                self._classes[class_name] = klass
+        return self._classes
+
+    @property
+    def ui(self):
+        if not hasattr(self, '_ui'):
+            if 'UI/ui.yaml' in self.contents.namelist():
+                self._ui = self.contents.open('UI/ui.yaml')
+            else:
+                self._ui = None
+        return self._ui
+
+    @property
+    def logo(self):
+        if not hasattr(self, '_logo'):
+            if 'logo.png' in self.contents.namelist():
+                self._logo = self.contents.open('logo.png')
+            else:
+                self._logo = None
+        return self._logo
+
+    def _get_package_order(self, packages_graph):
+        """Sorts packages according to dependencies between them
+
+        Murano allows cyclic dependencies. It is impossible
+        to do topological sort for graph with cycles, so at first
+        graph condensation should be built.
+        For condensation building Kosaraju's algorithm is used.
+        Packages in strongly connected components can be situated
+        in random order to each other.
+        """
+        def topological_sort(graph, start_node):
+            order = []
+            not_seen = set(graph)
+
+            def dfs(node):
+                not_seen.discard(node)
+                for dep_node in graph[node]:
+                    if dep_node in not_seen:
+                        dfs(dep_node)
+                order.append(node)
+
+            dfs(start_node)
+            return order
+
+        def transpose_graph(graph):
+            transposed = collections.defaultdict(list)
+            for node, deps in six.viewitems(graph):
+                for dep in deps:
+                    transposed[dep].append(node)
+            return transposed
+
+        order = topological_sort(packages_graph, self.manifest['FullName'])
+        order.reverse()
+        transposed = transpose_graph(packages_graph)
+
+        def top_sort_by_components(graph, component_order):
+            result = []
+            seen = set()
+
+            def dfs(node):
+                seen.add(node)
+                result.append(node)
+                for dep_node in graph[node]:
+                    if dep_node not in seen:
+                        dfs(dep_node)
+            for item in component_order:
+                if item not in seen:
+                    dfs(item)
+            return reversed(result)
+        return top_sort_by_components(transposed, order)
+
+    def requirements(self, base_url, path=None, dep_dict=None):
+        """Scans Require section of manifests of all the dependencies.
+
+        Returns a dict with FQPNs as keys and respective Package objects
+        as values, ordered by topological sort.
+
+        :param base_url: url of packages location
+        :param path: local path of packages location
+        :param dep_dict: unused. Left for backward compatibility
+        """
+
+        unordered_requirements = {}
+        requirements_graph = collections.defaultdict(list)
+        dep_queue = collections.deque([(self.manifest['FullName'], self)])
+        while dep_queue:
+            dep_name, dep_file = dep_queue.popleft()
+            unordered_requirements[dep_name] = dep_file
+            direct_deps = Package._get_direct_deps(dep_file, base_url, path)
+            for name, file in direct_deps:
+                if name not in unordered_requirements:
+                    dep_queue.append((name, file))
+            requirements_graph[dep_name] = [dep[0] for dep in direct_deps]
+
+        ordered_reqs_names = self._get_package_order(requirements_graph)
+        ordered_reqs_dict = collections.OrderedDict()
+        for name in ordered_reqs_names:
+            ordered_reqs_dict[name] = unordered_requirements[name]
+
+        return ordered_reqs_dict
+
+    @staticmethod
+    def _get_direct_deps(package, base_url, path):
+        result = []
+        if 'Require' in package.manifest:
+            for dep_name, ver in six.iteritems(package.manifest['Require']):
+                try:
+                    req_file = Package.from_location(
+                        dep_name,
+                        version=ver,
+                        path=path,
+                        base_url=base_url,
+                    )
+                except Exception as e:
+                    LOG.error("Error {0} occurred while parsing package {1}, "
+                              "required by {2} package".format(
+                                  e, dep_name,
+                                  package.manifest['FullName']))
+                    continue
+                result.append((req_file.manifest['FullName'], req_file))
+        return result
+
+
+class NamespaceResolver(object):
+    """Copied from main murano repo
+
+    original at murano/dsl/namespace_resolver.py
+    """
+
+    def __init__(self, namespaces):
+        self._namespaces = namespaces
+        self._namespaces[''] = ''
+
+    def resolve_name(self, name, relative=None):
+        if name is None:
+            raise ValueError()
+        if name and name.startswith(':'):
+            return name[1:]
+        if ':' in name:
+            parts = name.split(':')
+            if len(parts) != 2 or not parts[1]:
+                raise NameError('Incorrectly formatted name ' + name)
+            if parts[0] not in self._namespaces:
+                raise KeyError('Unknown namespace prefix ' + parts[0])
+            return '.'.join((self._namespaces[parts[0]], parts[1]))
+        if not relative and '=' in self._namespaces and '.' not in name:
+            return '.'.join((self._namespaces['='], name))
+        if relative and '.' not in name:
+            return '.'.join((relative, name))
+        return name
+
+
+def get_local_inheritance(classes):
+    result = {}
+    for class_name, klass in six.iteritems(classes):
+        if 'Extends' not in klass:
+            continue
+        ns = klass.get('Namespaces')
+        if ns:
+            resolver = NamespaceResolver(ns)
+        else:
+            resolver = None
+
+        if isinstance(klass['Extends'], list):
+            bases = klass['Extends']
+        else:
+            bases = [klass['Extends']]
+        for base_class in bases:
+            if resolver:
+                base_fqn = resolver.resolve_name(base_class)
+            else:
+                base_fqn = base_class
+            result.setdefault(base_fqn, []).append(class_name)
+    return result