[packetary] Repository class

class Repository composes from:
* RepositryDriver - low-level support for physical repository. deb, yum, etc.
* RepositoryController - infrastcuture method to communicate with driver
* RepositoryApi - high-level class, that provides methods to work with repository

Change-Id: Iaf868fca982d91089e369d13a6fb381ff879ea73
Implements: blueprint refactor-local-mirror-scripts
Partial-Bug: #1487077

packetary/__init__.py

@@ -14,9 +14,18 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-
 import pbr.version
 
+from packetary.api import Configuration
+from packetary.api import Context
+from packetary.api import RepositoryApi
+
+
+__all__ = [
+    "Configuration",
+    "Context",
+    "RepositoryApi",
+]
 
 __version__ = pbr.version.VersionInfo(
     'packetary').version_string()

packetary/api.py

@@ -0,0 +1,217 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging
+
+import six
+
+from packetary.controllers import RepositoryController
+from packetary.library.connections import ConnectionsManager
+from packetary.library.executor import AsynchronousSection
+from packetary.objects import Index
+from packetary.objects import PackageRelation
+from packetary.objects import PackagesTree
+from packetary.objects.statistics import CopyStatistics
+
+
+logger = logging.getLogger(__package__)
+
+
+class Configuration(object):
+    """The configuration holder."""
+
+    def __init__(self, http_proxy=None, https_proxy=None,
+                 retries_num=0, threads_num=0,
+                 ignore_errors_num=0):
+        """Initialises.
+
+        :param http_proxy: the url of proxy for connections over http,
+                           no-proxy will be used if it is not specified
+        :param https_proxy: the url of proxy for connections over https,
+                            no-proxy will be used if it is not specified
+        :param retries_num: the number of retries on errors
+        :param threads_num: the max number of active threads
+        :param ignore_errors_num: the number of errors that may occurs
+                before stop processing
+        """
+
+        self.http_proxy = http_proxy
+        self.https_proxy = https_proxy
+        self.ignore_errors_num = ignore_errors_num
+        self.retries_num = retries_num
+        self.threads_num = threads_num
+
+
+class Context(object):
+    """The infra-objects holder."""
+
+    def __init__(self, config):
+        """Initialises.
+
+        :param config: the configuration
+        """
+        self._connection = ConnectionsManager(
+            proxy=config.http_proxy,
+            secure_proxy=config.https_proxy,
+            retries_num=config.retries_num
+        )
+        self._threads_num = config.threads_num
+        self._ignore_errors_num = config.ignore_errors_num
+
+    @property
+    def connection(self):
+        """Gets the connection."""
+        return self._connection
+
+    def async_section(self, ignore_errors_num=None):
+        """Gets the execution scope.
+
+        :param ignore_errors_num: custom value for ignore_errors_num,
+                                  the class value is used if omitted.
+        """
+        if ignore_errors_num is None:
+            ignore_errors_num = self._ignore_errors_num
+
+        return AsynchronousSection(self._threads_num, ignore_errors_num)
+
+
+class RepositoryApi(object):
+    """Provides high-level API to operate with repositories."""
+
+    def __init__(self, controller):
+        """Initialises.
+
+        :param controller: the repository controller.
+        """
+        self.controller = controller
+
+    @classmethod
+    def create(cls, config, repotype, repoarch):
+        """Creates the repository API instance.
+
+        :param config: the configuration
+        :param repotype: the kind of repository(deb, yum, etc)
+        :param repoarch: the architecture of repository (x86_64 or i386)
+        """
+        context = config if isinstance(config, Context) else Context(config)
+        return cls(RepositoryController.load(context, repotype, repoarch))
+
+    def get_packages(self, origin, debs=None, requirements=None):
+        """Gets the list of packages from repository(es).
+
+        :param origin: The list of repository`s URLs
+        :param debs: the list of repository`s URL to calculate list of
+                     dependencies, that will be used to filter packages.
+        :param requirements: the list of package relations,
+                        to resolve the list of mandatory packages.
+        :return: the set of packages
+        """
+        repositories = self._get_repositories(origin)
+        return self._get_packages(repositories, debs, requirements)
+
+    def clone_repositories(self, origin, destination, debs=None,
+                           requirements=None, keep_existing=True,
+                           include_source=False, include_locale=False):
+        """Creates the clones of specified repositories in local folder.
+
+        :param origin: The list of repository`s URLs
+        :param destination: the destination folder path
+        :param debs: the list of repository`s URL to calculate list of
+                     dependencies, that will be used to filter packages.
+        :param requirements: the list of package relations,
+                        to resolve the list of mandatory packages.
+        :param keep_existing: If False - local packages that does not exist
+                              in original repo will be removed.
+        :param include_source: if True, the source packages
+                               will be copied as well.
+        :param include_locale: if True, the locales
+                               will be copied as well.
+        :return: count of copied and total packages.
+        """
+        repositories = self._get_repositories(origin)
+        packages = self._get_packages(repositories, debs, requirements)
+        mirrors = self.controller.clone_repositories(
+            repositories, destination, include_source, include_locale
+        )
+
+        package_groups = dict((x, set()) for x in repositories)
+        for pkg in packages:
+            package_groups[pkg.repository].add(pkg)
+
+        stat = CopyStatistics()
+        for repo, packages in six.iteritems(package_groups):
+            mirror = mirrors[repo]
+            logger.info("copy packages from - %s", repo)
+            self.controller.copy_packages(
+                mirror, packages, keep_existing, stat.on_package_copied
+            )
+        return stat
+
+    def get_unresolved_dependencies(self, urls):
+        """Gets list of unresolved dependencies for repository(es).
+
+        :param urls: The list of repository`s URLs
+        :return: list of unresolved dependencies
+        """
+        packages = PackagesTree()
+        self.controller.load_packages(
+            self._get_repositories(urls),
+            packages.add
+        )
+        return packages.get_unresolved_dependencies()
+
+    def _get_repositories(self, urls):
+        """Gets the set of repositories by url."""
+        repositories = set()
+        self.controller.load_repositories(urls, repositories.add)
+        return repositories
+
+    def _get_packages(self, repositories, master, requirements):
+        """Gets the list of packages according to master and requirements."""
+        if master is None and requirements is None:
+            packages = set()
+            self.controller.load_packages(repositories, packages.add)
+            return packages
+
+        packages = PackagesTree()
+        self.controller.load_packages(repositories, packages.add)
+        if master is not None:
+            main_index = Index()
+            self.controller.load_packages(
+                self._get_repositories(master),
+                main_index.add
+            )
+        else:
+            main_index = None
+
+        return packages.get_minimal_subset(
+            main_index,
+            self._parse_requirements(requirements)
+        )
+
+    @staticmethod
+    def _parse_requirements(requirements):
+        """Gets the list of relations from requirements.
+
+        :param requirements: the list of requirement in next format:
+                             'name [cmp version]|[alt [cmp version]]'
+        """
+        if requirements is not None:
+            return set(
+                PackageRelation.from_args(
+                    *(x.split() for x in r.split("|"))) for r in requirements
+            )
+        return set()

packetary/controllers/__init__.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from packetary.controllers.repository import RepositoryController
+
+__all__ = [
+    "RepositoryController"
+]

packetary/controllers/repository.py

@@ -0,0 +1,169 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging
+import os
+
+import six
+import stevedore
+
+
+logger = logging.getLogger(__package__)
+
+urljoin = six.moves.urllib.parse.urljoin
+
+
+class RepositoryController(object):
+    """Implements low-level functionality to communicate with drivers."""
+
+    _drivers = None
+
+    def __init__(self, context, driver, arch):
+        self.context = context
+        self.driver = driver
+        self.arch = arch
+
+    @classmethod
+    def load(cls, context, driver_name, repoarch):
+        """Creates the repository manager.
+
+        :param context: the context
+        :param driver_name: the name of required driver
+        :param repoarch: the architecture of repository (x86_64 or i386)
+        """
+        if cls._drivers is None:
+            cls._drivers = stevedore.ExtensionManager(
+                "packetary.drivers", invoke_on_load=True
+            )
+        try:
+            driver = cls._drivers[driver_name].obj
+        except KeyError:
+            raise NotImplementedError(
+                "The driver {0} is not supported yet.".format(driver_name)
+            )
+        return cls(context, driver, repoarch)
+
+    def load_repositories(self, urls, consumer):
+        """Loads the repository objects from url.
+
+        :param urls: the list of repository urls.
+        :param consumer: the callback to consume objects
+        """
+        if isinstance(urls, six.string_types):
+            urls = [urls]
+
+        connection = self.context.connection
+        for parsed_url in self.driver.parse_urls(urls):
+            self.driver.get_repository(
+                connection, parsed_url, self.arch, consumer
+            )
+
+    def load_packages(self, repositories, consumer):
+        """Loads packages from repository.
+
+        :param repositories: the repository object
+        :param consumer: the callback to consume objects
+        """
+        connection = self.context.connection
+        for r in repositories:
+            self.driver.get_packages(connection, r, consumer)
+
+    def assign_packages(self, repository, packages, keep_existing=True):
+        """Assigns new packages to the repository.
+
+         It replaces the current repository`s packages.
+
+        :param repository: the target repository
+        :param packages: the set of new packages
+        :param keep_existing:
+            if True, all existing packages will be kept as is.
+            if False, all existing packages, that are not included
+            to new packages will be removed.
+        """
+
+        if not isinstance(packages, set):
+            packages = set(packages)
+        else:
+            packages = packages.copy()
+
+        if keep_existing:
+            consume_exist = packages.add
+        else:
+            def consume_exist(package):
+                if package not in packages:
+                    filepath = os.path.join(
+                        package.repository.url, package.filename
+                    )
+                    logger.info("remove package - %s.", filepath)
+                    os.remove(filepath)
+
+        self.driver.get_packages(
+            self.context.connection, repository, consume_exist
+        )
+        self.driver.rebuild_repository(repository, packages)
+
+    def copy_packages(self, repository, packages, keep_existing, observer):
+        """Copies packages to repository.
+
+        :param repository: the target repository
+        :param packages: the set of packages
+        :param keep_existing: see assign_packages for more details
+        :param observer: the package copying process observer
+        """
+        with self.context.async_section() as section:
+            for package in packages:
+                section.execute(
+                    self._copy_package, repository, package, observer
+                )
+        self.assign_packages(repository, packages, keep_existing)
+
+    def clone_repositories(self, repositories, destination,
+                           source=False, locale=False):
+        """Creates copy of repositories.
+
+        :param repositories: the origin repositories
+        :param destination: the target folder
+        :param source: If True, the source packages will be copied too.
+        :param locale: If True, the localisation will be copied too.
+        :return: the mapping origin to cloned repository.
+        """
+        mirros = dict()
+        destination = os.path.abspath(destination)
+        with self.context.async_section(0) as section:
+            for r in repositories:
+                section.execute(
+                    self._clone_repository,
+                    r, destination, source, locale, mirros
+                )
+        return mirros
+
+    def _clone_repository(self, r, destination, source, locale, mirrors):
+        """Creates clone of repository and stores it in mirrors."""
+        clone = self.driver.clone_repository(
+            self.context.connection, r, destination, source, locale
+        )
+        mirrors[r] = clone
+
+    def _copy_package(self, target, package, observer):
+        """Synchronises remote file to local fs."""
+        dst_path = os.path.join(target.url, package.filename)
+        src_path = urljoin(package.repository.url, package.filename)
+        bytes_copied = self.context.connection.retrieve(
+            src_path, dst_path, size=package.filesize
+        )
+        if package.filesize < 0:
+            package.filesize = bytes_copied
+        observer(bytes_copied)

packetary/drivers/base.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import abc
+import logging
+
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class RepositoryDriverBase(object):
+    """The super class for Repository Drivers.
+
+    For implementing support of new type of repository:
+    - inherit this class
+    - implement all abstract methods
+    - register implementation in 'packetary.drivers' namespace
+    """
+    def __init__(self):
+        self.logger = logging.getLogger(__package__)
+
+    @abc.abstractmethod
+    def parse_urls(self, urls):
+        """Parses the repository url.
+
+        :return: the sequence of parsed urls
+        """
+
+    @abc.abstractmethod
+    def get_repository(self, connection, url, arch, consumer):
+        """Loads the repository meta information from URL.
+
+        :param connection: the connection manager instance
+        :param url: the repository`s url
+        :param arch: the repository`s architecture
+        :param consumer: the callback to consume result
+        """
+
+    @abc.abstractmethod
+    def get_packages(self, connection, repository, consumer):
+        """Loads packages from repository.
+
+        :param connection: the connection manager instance
+        :param repository: the repository object
+        :param consumer: the callback to consume result
+        """
+
+    @abc.abstractmethod
+    def clone_repository(self, connection, repository, destination,
+                         source=False, locale=False):
+        """Creates copy of repository.
+
+        :param connection: the connection manager instance
+        :param repository: the source repository
+        :param destination: the destination folder
+        :param source: copy source files
+        :param locale: copy localisation
+        :return: The copy of repository
+        """
+
+    @abc.abstractmethod
+    def rebuild_repository(self, repository, packages):
+        """Re-builds the repository.
+
+        :param repository: the target repository
+        :param packages: the set of packages
+        """

packetary/objects/__init__.py

@@ -20,6 +20,7 @@ from packetary.objects.package import FileChecksum
 from packetary.objects.package import Package
 from packetary.objects.package_relation import PackageRelation
 from packetary.objects.package_relation import VersionRange
+from packetary.objects.packages_tree import PackagesTree
 from packetary.objects.repository import Repository
 
 
@@ -28,6 +29,7 @@ __all__ = [
     "Index",
     "Package",
     "PackageRelation",
+    "PackagesTree",
     "Repository",
     "VersionRange",
 ]

packetary/objects/packages_tree.py

@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import warnings
+
+from packetary.objects.index import Index
+
+
+class UnresolvedWarning(UserWarning):
+    """Warning about unresolved depends."""
+    pass
+
+
+class PackagesTree(Index):
+    """Helper class to deal with dependency graph."""
+
+    def __init__(self):
+        super(PackagesTree, self).__init__()
+        self.mandatory_packages = []
+
+    def add(self, package):
+        super(PackagesTree, self).add(package)
+        # store all mandatory packages in separated list for quick access
+        if package.mandatory:
+            self.mandatory_packages.append(package)
+
+    def get_unresolved_dependencies(self, unresolved=None):
+        """Gets the set of unresolved dependencies.
+
+        :param unresolved: the known list of unresolved packages.
+        :return: the set of unresolved depends.
+        """
+        return self.__get_unresolved_dependencies(self)
+
+    def get_minimal_subset(self, main, requirements):
+        """Gets the minimal work subset.
+
+        :param main: the main index, to complete requirements.
+        :param requirements: additional requirements.
+        :return: The set of resolved depends.
+        """
+
+        unresolved = set()
+        resolved = set()
+        if main is None:
+            def pkg_filter(*_):
+                pass
+        else:
+            pkg_filter = main.find
+            self.__get_unresolved_dependencies(main, requirements)
+
+        stack = list()
+        stack.append((None, requirements))
+
+        # add all mandatory packages
+        for pkg in self.mandatory_packages:
+            stack.append((pkg, pkg.requires))
+
+        while len(stack) > 0:
+            pkg, required = stack.pop()
+            resolved.add(pkg)
+            for require in required:
+                for rel in require:
+                    if rel not in unresolved:
+                        if pkg_filter(rel.name, rel.version) is not None:
+                            break
+                        # use all packages that meets depends
+                        candidates = self.find_all(rel.name, rel.version)
+                        found = False
+                        for cand in candidates:
+                            if cand == pkg:
+                                continue
+                            found = True
+                            if cand not in resolved:
+                                stack.append((cand, cand.requires))
+
+                        if found:
+                            break
+                else:
+                    unresolved.add(require)
+                    msg = "Unresolved depends: {0}".format(require)
+                    warnings.warn(UnresolvedWarning(msg))
+
+        resolved.remove(None)
+        return resolved
+
+    @staticmethod
+    def __get_unresolved_dependencies(index, unresolved=None):
+        """Gets the set of unresolved dependencies.
+
+        :param index: the search index.
+        :param unresolved: the known list of unresolved packages.
+        :return: the set of unresolved depends.
+        """
+
+        if unresolved is None:
+            unresolved = set()
+
+        for pkg in index:
+            for require in pkg.requires:
+                for rel in require:
+                    if rel not in unresolved:
+                        candidate = index.find(rel.name, rel.version)
+                        if candidate is not None and candidate != pkg:
+                            break
+                else:
+                    unresolved.add(require)
+        return unresolved

packetary/objects/statistics.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import copy
+
+
+class CopyStatistics(object):
+    """The statistics of packages copying"""
+    def __init__(self):
+        # the number of copied packages
+        self.copied = 0
+        # the number of total packages
+        self.total = 0
+
+    def on_package_copied(self, bytes_copied):
+        """Proceed next copied package."""
+        if bytes_copied > 0:
+            self.copied += 1
+        self.total += 1
+
+    def __iadd__(self, other):
+        if not isinstance(other, CopyStatistics):
+            raise TypeError
+
+        self.copied += other.copied
+        self.total += other.total
+        return self
+
+    def __add__(self, other):
+        result = copy.copy(self)
+        result += other
+        return result

packetary/tests/stubs/executor.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+class Executor(object):
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_):
+        return False
+
+    @staticmethod
+    def execute(f, *args, **kwargs):
+        return f(*args, **kwargs)

packetary/tests/stubs/helpers.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+
+class CallbacksAdapter(mock.MagicMock):
+    """Helper to return data through callback."""
+
+    def __call__(self, *args, **kwargs):
+        if len(args) > 0:
+            callback = args[-1]
+        else:
+            callback = None
+
+        if not callable(callback):
+            return super(CallbacksAdapter, self).__call__(*args, **kwargs)
+
+        args = args[:-1]
+        data = super(CallbacksAdapter, self).__call__(*args, **kwargs)
+
+        if isinstance(data, list):
+            for d in data:
+                callback(d)
+        else:
+            callback(data)

packetary/tests/test_packages_tree.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import warnings
+
+from packetary.objects import Index
+from packetary.objects import PackagesTree
+from packetary.tests import base
+from packetary.tests.stubs import generator
+
+
+class TestPackagesTree(base.TestCase):
+    def setUp(self):
+        super(TestPackagesTree, self).setUp()
+
+    def test_get_unresolved_dependencies(self):
+        ptree = PackagesTree()
+        ptree.add(generator.gen_package(
+            1, requires=[generator.gen_relation("unresolved")]))
+        ptree.add(generator.gen_package(2, requires=None))
+        ptree.add(generator.gen_package(
+            3, requires=[generator.gen_relation("package1")]
+        ))
+        ptree.add(generator.gen_package(
+            4,
+            requires=[generator.gen_relation("loop")],
+            obsoletes=[generator.gen_relation("loop", ["le", 1])]
+        ))
+
+        unresolved = ptree.get_unresolved_dependencies()
+        self.assertItemsEqual(
+            ["loop", "unresolved"],
+            (x.name for x in unresolved)
+        )
+
+    def test_get_minimal_subset_with_master(self):
+        ptree = PackagesTree()
+        ptree.add(generator.gen_package(1, requires=None))
+        ptree.add(generator.gen_package(2, requires=None))
+        ptree.add(generator.gen_package(3, requires=None))
+        ptree.add(generator.gen_package(
+            4, requires=[generator.gen_relation("package1")]
+        ))
+
+        master = Index()
+        master.add(generator.gen_package(1, requires=None))
+        master.add(generator.gen_package(
+            5,
+            requires=[generator.gen_relation(
+                "package10",
+                alternative=generator.gen_relation("package4")
+            )]
+        ))
+
+        unresolved = set([generator.gen_relation("package3")])
+        resolved = ptree.get_minimal_subset(master, unresolved)
+        self.assertItemsEqual(
+            ["package3", "package4"],
+            (x.name for x in resolved)
+        )
+
+    def test_get_minimal_subset_without_master(self):
+        ptree = PackagesTree()
+        ptree.add(generator.gen_package(1, requires=None))
+        ptree.add(generator.gen_package(2, requires=None))
+        ptree.add(generator.gen_package(
+            3, requires=[generator.gen_relation("package1")]
+        ))
+        unresolved = set([generator.gen_relation("package3")])
+        resolved = ptree.get_minimal_subset(None, unresolved)
+        self.assertItemsEqual(
+            ["package3", "package1"],
+            (x.name for x in resolved)
+        )
+
+    def test_mandatory_packages_always_included(self):
+        ptree = PackagesTree()
+        ptree.add(generator.gen_package(1, requires=None, mandatory=True))
+        ptree.add(generator.gen_package(2, requires=None))
+        ptree.add(generator.gen_package(3, requires=None))
+        unresolved = set([generator.gen_relation("package3")])
+        resolved = ptree.get_minimal_subset(None, unresolved)
+        self.assertItemsEqual(
+            ["package3", "package1"],
+            (x.name for x in resolved)
+        )
+
+    def test_warning_if_unresolved(self):
+        ptree = PackagesTree()
+        ptree.add(generator.gen_package(
+            1, requires=None))
+
+        with warnings.catch_warnings(record=True) as log:
+            ptree.get_minimal_subset(
+                None, [generator.gen_relation("package2")]
+            )
+        self.assertIn("package2", str(log[0]))

packetary/tests/test_repository_api.py

@@ -0,0 +1,230 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from packetary.api import Configuration
+from packetary.api import Context
+from packetary.api import RepositoryApi
+from packetary.tests import base
+from packetary.tests.stubs import generator
+from packetary.tests.stubs.helpers import CallbacksAdapter
+
+
+class TestRepositoryApi(base.TestCase):
+    def test_get_packages_as_is(self):
+        controller = CallbacksAdapter()
+        pkg = generator.gen_package(name="test")
+        controller.load_packages.side_effect = [
+            pkg
+        ]
+        api = RepositoryApi(controller)
+        packages = api.get_packages("file:///repo1")
+        self.assertEqual(1, len(packages))
+        package = packages.pop()
+        self.assertIs(pkg, package)
+
+    def test_get_packages_with_depends_resolving(self):
+        controller = CallbacksAdapter()
+        controller.load_packages.side_effect = [
+            [
+                generator.gen_package(idx=1, requires=None),
+                generator.gen_package(
+                    idx=2, requires=[generator.gen_relation("package1")]
+                ),
+                generator.gen_package(
+                    idx=3, requires=[generator.gen_relation("package1")]
+                ),
+                generator.gen_package(idx=4, requires=None),
+                generator.gen_package(idx=5, requires=None),
+            ],
+            generator.gen_package(
+                idx=6, requires=[generator.gen_relation("package2")]
+            ),
+        ]
+
+        api = RepositoryApi(controller)
+        packages = api.get_packages([
+            "file:///repo1", "file:///repo2"
+        ],
+            "file:///repo3", ["package4"]
+        )
+
+        self.assertEqual(3, len(packages))
+        self.assertItemsEqual(
+            ["package1", "package4", "package2"],
+            (x.name for x in packages)
+        )
+        controller.load_repositories.assert_any_call(
+            ["file:///repo1", "file:///repo2"]
+        )
+        controller.load_repositories.assert_any_call(
+            "file:///repo3"
+        )
+
+    def test_clone_repositories_as_is(self):
+        controller = CallbacksAdapter()
+        repo = generator.gen_repository(name="repo1")
+        packages = [
+            generator.gen_package(name="test1", repository=repo),
+            generator.gen_package(name="test2", repository=repo)
+        ]
+        mirror = generator.gen_repository(name="mirror")
+        controller.load_repositories.return_value = repo
+        controller.load_packages.return_value = packages
+        controller.clone_repositories.return_value = {repo: mirror}
+        controller.copy_packages.return_value = [0, 1]
+        api = RepositoryApi(controller)
+        stats = api.clone_repositories(
+            ["file:///repo1"], "/mirror", keep_existing=True
+        )
+        self.assertEqual(2, stats.total)
+        self.assertEqual(1, stats.copied)
+        controller.copy_packages.assert_called_once_with(
+            mirror, set(packages), True
+        )
+
+    def test_copy_minimal_subset_of_repository(self):
+        controller = CallbacksAdapter()
+        repo1 = generator.gen_repository(name="repo1")
+        repo2 = generator.gen_repository(name="repo2")
+        repo3 = generator.gen_repository(name="repo3")
+        mirror1 = generator.gen_repository(name="mirror1")
+        mirror2 = generator.gen_repository(name="mirror2")
+        pkg_group1 = [
+            generator.gen_package(
+                idx=1, requires=None, repository=repo1
+            ),
+            generator.gen_package(
+                idx=1, version=2, requires=None, repository=repo1
+            ),
+            generator.gen_package(
+                idx=2, requires=None, repository=repo1
+            )
+        ]
+        pkg_group2 = [
+            generator.gen_package(
+                idx=4,
+                requires=[generator.gen_relation("package1")],
+                repository=repo2,
+                mandatory=True,
+            )
+        ]
+        pkg_group3 = [
+            generator.gen_package(
+                idx=3, requires=None, repository=repo1
+            )
+        ]
+        controller.load_repositories.side_effect = [[repo1, repo2], repo3]
+        controller.load_packages.side_effect = [
+            pkg_group1 + pkg_group2 + pkg_group3,
+            generator.gen_package(
+                idx=6,
+                repository=repo3,
+                requires=[generator.gen_relation("package2")]
+            )
+        ]
+        controller.clone_repositories.return_value = {
+            repo1: mirror1, repo2: mirror2
+        }
+        controller.copy_packages.return_value = 1
+        api = RepositoryApi(controller)
+        api.clone_repositories(
+            ["file:///repo1", "file:///repo2"], "/mirror",
+            ["file:///repo3"],
+            keep_existing=True
+        )
+        controller.copy_packages.assert_any_call(
+            mirror1, set(pkg_group1), True
+        )
+        controller.copy_packages.assert_any_call(
+            mirror2, set(pkg_group2), True
+        )
+        self.assertEqual(2, controller.copy_packages.call_count)
+
+    def test_get_unresolved(self):
+        controller = CallbacksAdapter()
+        pkg = generator.gen_package(
+            name="test", requires=[generator.gen_relation("test2")]
+        )
+        controller.load_packages.side_effect = [
+            pkg
+        ]
+        api = RepositoryApi(controller)
+        r = api.get_unresolved_dependencies("file:///repo1")
+        controller.load_repositories.assert_called_once_with("file:///repo1")
+        self.assertItemsEqual(
+            ["test2"],
+            (x.name for x in r)
+        )
+
+    def test_parse_requirements(self):
+        requirements = RepositoryApi._parse_requirements(
+            ["p1 le 2 | p2 | p3 ge 2"]
+        )
+
+        expected = generator.gen_relation(
+            "p1",
+            ["le", '2'],
+            generator.gen_relation(
+                "p2",
+                None,
+                generator.gen_relation(
+                    "p3",
+                    ["ge", '2']
+                )
+            )
+        )
+        self.assertEqual(1, len(requirements))
+        self.assertEqual(
+            list(expected),
+            list(requirements.pop())
+        )
+
+
+class TestContext(base.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.config = Configuration(
+            threads_num=2,
+            ignore_errors_num=3,
+            retries_num=5,
+            http_proxy="http://localhost",
+            https_proxy="https://localhost"
+        )
+
+    @mock.patch("packetary.api.ConnectionsManager")
+    def test_initialise_connection_manager(self, conn_manager):
+        context = Context(self.config)
+        conn_manager.assert_called_once_with(
+            proxy="http://localhost",
+            secure_proxy="https://localhost",
+            retries_num=5
+        )
+
+        self.assertIs(
+            conn_manager(),
+            context.connection
+        )
+
+    @mock.patch("packetary.api.AsynchronousSection")
+    def test_asynchronous_section(self, async_section):
+        context = Context(self.config)
+        s = context.async_section()
+        async_section.assert_called_with(2, 3)
+        self.assertIs(s, async_section())
+        context.async_section(0)
+        async_section.assert_called_with(2, 0)

packetary/tests/test_repository_contoller.py

@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import copy
+import mock
+import six
+
+from packetary.controllers import RepositoryController
+from packetary.tests import base
+from packetary.tests.stubs.executor import Executor
+from packetary.tests.stubs.generator import gen_package
+from packetary.tests.stubs.generator import gen_repository
+from packetary.tests.stubs.helpers import CallbacksAdapter
+
+
+class TestRepositoryController(base.TestCase):
+    def setUp(self):
+        self.driver = mock.MagicMock()
+        self.context = mock.MagicMock()
+        self.context.async_section.return_value = Executor()
+        self.ctrl = RepositoryController(self.context, self.driver, "x86_64")
+
+    def test_load_fail_if_unknown_driver(self):
+        with self.assertRaisesRegexp(NotImplementedError, "unknown_driver"):
+            RepositoryController.load(
+                self.context,
+                "unknown_driver",
+                "x86_64"
+            )
+
+    @mock.patch("packetary.controllers.repository.stevedore")
+    def test_load_driver(self, stevedore):
+        stevedore.ExtensionManager.return_value = {
+            "test": mock.MagicMock(obj=self.driver)
+        }
+        RepositoryController._drivers = None
+        controller = RepositoryController.load(self.context, "test", "x86_64")
+        self.assertIs(self.driver, controller.driver)
+
+    def test_load_repositories(self):
+        self.driver.parse_urls.return_value = ["test1"]
+        consumer = mock.MagicMock()
+        self.ctrl.load_repositories("file:///test1", consumer)
+        self.driver.parse_urls.assert_called_once_with(["file:///test1"])
+        self.driver.get_repository.assert_called_once_with(
+            self.context.connection, "test1", "x86_64", consumer
+        )
+        for url in [six.u("file:///test1"), ["file:///test1"]]:
+            self.driver.reset_mock()
+            self.ctrl.load_repositories(url, consumer)
+            if not isinstance(url, list):
+                url = [url]
+            self.driver.parse_urls.assert_called_once_with(url)
+
+    def test_load_packages(self):
+        repo = mock.MagicMock()
+        consumer = mock.MagicMock()
+        self.ctrl.load_packages([repo], consumer)
+        self.driver.get_packages.assert_called_once_with(
+            self.context.connection, repo, consumer
+        )
+
+    @mock.patch("packetary.controllers.repository.os")
+    def test_assign_packages(self, os):
+        repo = gen_repository(url="/test/repo")
+        packages = [
+            gen_package(name="test1", repository=repo),
+            gen_package(name="test2", repository=repo)
+        ]
+        existed_packages = [
+            gen_package(name="test3", repository=repo),
+            gen_package(name="test2", repository=repo)
+        ]
+
+        os.path.join = lambda *x: "/".join(x)
+        self.driver.get_packages = CallbacksAdapter()
+        self.driver.get_packages.return_value = existed_packages
+        self.ctrl.assign_packages(repo, packages, True)
+        os.remove.assert_not_called()
+        all_packages = set(packages + existed_packages)
+        self.driver.rebuild_repository.assert_called_once_with(
+            repo, all_packages
+        )
+        self.driver.rebuild_repository.reset_mock()
+        self.ctrl.assign_packages(repo, packages, False)
+        self.driver.rebuild_repository.assert_called_once_with(
+            repo, set(packages)
+        )
+        os.remove.assert_called_once_with("/test/repo/test3.pkg")
+
+    def test_copy_packages(self):
+        repo = gen_repository(url="file:///repo/")
+        packages = [
+            gen_package(name="test1", repository=repo, filesize=10),
+            gen_package(name="test2", repository=repo, filesize=-1)
+        ]
+        target = gen_repository(url="/test/repo")
+        self.context.connection.retrieve.side_effect = [0, 10]
+        observer = mock.MagicMock()
+        self.ctrl.copy_packages(target, packages, True, observer)
+        observer.assert_has_calls([mock.call(0), mock.call(10)])
+        self.context.connection.retrieve.assert_any_call(
+            "file:///repo/test1.pkg",
+            "/test/repo/test1.pkg",
+            size=10
+        )
+        self.context.connection.retrieve.assert_any_call(
+            "file:///repo/test2.pkg",
+            "/test/repo/test2.pkg",
+            size=-1
+        )
+        self.driver.rebuild_repository.assert_called_once_with(
+            target, set(packages)
+        )
+
+    @mock.patch("packetary.controllers.repository.os")
+    def test_clone_repository(self, os):
+        os.path.abspath.return_value = "/root/repo"
+        repos = [
+            gen_repository(name="test1"),
+            gen_repository(name="test2")
+        ]
+        clones = [copy.copy(x) for x in repos]
+        self.driver.clone_repository.side_effect = clones
+        mirrors = self.ctrl.clone_repositories(repos, "./repo")
+        for r in repos:
+            self.driver.clone_repository.assert_any_call(
+                self.context.connection, r, "/root/repo", False, False
+            )
+        self.assertEqual(mirrors, dict(zip(repos, clones)))

requirements.txt

@@ -7,4 +7,5 @@ Babel>=1.3
 eventlet>=0.15
 bintrees>=2.0.2
 chardet>=2.3.0
+stevedore>=1.1.0
 six>=1.5.2