From d6610553222345429719cd727a5caa2e2b60d0eb Mon Sep 17 00:00:00 2001 From: Bulat Gaifullin Date: Sun, 24 Apr 2016 08:57:41 -0500 Subject: [PATCH] Introduced new scheme to declare requirements the requirements can contain the following sections: packages - the list of packages that are needed repositories - the list of repositories, packages from that are needed mandatory - boolean flag that uses to automatically copy mandatory packages Change-Id: Ic26f991c0bf1e9819005cd4bbe7ed40228b2ce1b --- packetary/__init__.py | 12 +- packetary/api.py | 328 ------------- .../__init__.py} | 26 +- packetary/api/context.py | 81 ++++ packetary/api/loaders.py | 108 +++++ .../packages_schema.py => api/options.py} | 29 +- packetary/api/repositories.py | 166 +++++++ packetary/{objects => api}/statistics.py | 0 packetary/api/validators.py | 111 +++++ packetary/cli/commands/base.py | 30 +- packetary/cli/commands/clone.py | 14 +- packetary/cli/commands/create.py | 4 +- packetary/cli/commands/packages.py | 11 +- packetary/cli/commands/unresolved.py | 4 +- packetary/controllers/repository.py | 7 +- packetary/drivers/base.py | 6 +- packetary/drivers/deb_driver.py | 3 +- packetary/drivers/rpm_driver.py | 3 +- packetary/library/functions.py | 26 + packetary/schemas/__init__.py | 6 +- packetary/schemas/requirements_schema.py | 74 +++ packetary/tests/base.py | 2 + packetary/tests/stubs/helpers.py | 6 +- packetary/tests/test_api_context.py | 57 +++ packetary/tests/test_api_loaders.py | 84 ++++ packetary/tests/test_api_validators.py | 91 ++++ packetary/tests/test_cli_commands.py | 27 +- packetary/tests/test_deb_driver.py | 2 +- packetary/tests/test_repository_api.py | 448 ++++++------------ packetary/tests/test_repository_contoller.py | 10 +- packetary/tests/test_rpm_driver.py | 6 +- packetary/tests/test_schemas.py | 93 ++-- 32 files changed, 1050 insertions(+), 825 deletions(-) delete mode 100644 packetary/api.py rename packetary/{schemas/package_filter_schema.py => api/__init__.py} (70%) create mode 100644 packetary/api/context.py create mode 100644 packetary/api/loaders.py rename packetary/{schemas/packages_schema.py => api/options.py} (58%) create mode 100644 packetary/api/repositories.py rename packetary/{objects => api}/statistics.py (100%) create mode 100644 packetary/api/validators.py create mode 100644 packetary/library/functions.py create mode 100644 packetary/schemas/requirements_schema.py create mode 100644 packetary/tests/test_api_context.py create mode 100644 packetary/tests/test_api_loaders.py create mode 100644 packetary/tests/test_api_validators.py diff --git a/packetary/__init__.py b/packetary/__init__.py index 2fca754..7954362 100644 --- a/packetary/__init__.py +++ b/packetary/__init__.py @@ -18,19 +18,13 @@ import pbr.version -from packetary.api import Configuration -from packetary.api import Context -from packetary.api import RepositoryApi +from packetary import api -__all__ = [ - "Configuration", - "Context", - "RepositoryApi", -] +__all__ = ["api", "__version__"] try: __version__ = pbr.version.VersionInfo( 'packetary').version_string() -except Exception as e: +except Exception: __version__ = "0.0.0" diff --git a/packetary/api.py b/packetary/api.py deleted file mode 100644 index bd83aeb..0000000 --- a/packetary/api.py +++ /dev/null @@ -1,328 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2015 Mirantis, Inc. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -from collections import defaultdict -import logging -import re - -import jsonschema -import six - -from packetary.controllers import RepositoryController -from packetary.library.connections import ConnectionsManager -from packetary.library.executor import AsynchronousSection -from packetary.objects import PackageRelation -from packetary.objects import PackagesForest -from packetary.objects import PackagesTree -from packetary.objects.statistics import CopyStatistics -from packetary.schemas import PACKAGE_FILES_SCHEMA -from packetary.schemas import PACKAGE_FILTER_SCHEMA -from packetary.schemas import PACKAGES_SCHEMA - -logger = logging.getLogger(__package__) - - -class Configuration(object): - """The configuration holder.""" - - def __init__(self, http_proxy=None, https_proxy=None, - retries_num=0, retry_interval=0, threads_num=0, - ignore_errors_num=0): - """Initialises. - - :param http_proxy: the url of proxy for connections over http, - no-proxy will be used if it is not specified - :param https_proxy: the url of proxy for connections over https, - no-proxy will be used if it is not specified - :param retries_num: the number of retries on errors - :param retry_interval: the minimal time between retries (in seconds) - :param threads_num: the max number of active threads - :param ignore_errors_num: the number of errors that may occurs - before stop processing - """ - - self.http_proxy = http_proxy - self.https_proxy = https_proxy - self.ignore_errors_num = ignore_errors_num - self.retries_num = retries_num - self.retry_interval = retry_interval - self.threads_num = threads_num - - -class Context(object): - """The infra-objects holder.""" - - def __init__(self, config): - """Initialises. - - :param config: the configuration - """ - self._connection = ConnectionsManager( - proxy=config.http_proxy, - secure_proxy=config.https_proxy, - retries_num=config.retries_num, - retry_interval=config.retry_interval - ) - self._threads_num = config.threads_num - self._ignore_errors_num = config.ignore_errors_num - - @property - def connection(self): - """Gets the connection.""" - return self._connection - - def async_section(self, ignore_errors_num=None): - """Gets the execution scope. - - :param ignore_errors_num: custom value for ignore_errors_num, - the class value is used if omitted. - """ - if ignore_errors_num is None: - ignore_errors_num = self._ignore_errors_num - - return AsynchronousSection(self._threads_num, ignore_errors_num) - - -class RepositoryApi(object): - """Provides high-level API to operate with repositories.""" - - def __init__(self, controller): - """Initialises. - - :param controller: the repository controller. - """ - self.controller = controller - - @classmethod - def create(cls, config, repotype, repoarch): - """Creates the repository API instance. - - :param config: the configuration - :param repotype: the kind of repository(deb, yum, etc) - :param repoarch: the architecture of repository (x86_64 or i386) - """ - context = config if isinstance(config, Context) else Context(config) - return cls(RepositoryController.load(context, repotype, repoarch)) - - def create_repository(self, repo_data, package_files): - """Create new repository with specified packages. - - :param repo_data: The description of repository - :param package_files: The list of URLs of packages - """ - self._validate_repo_data(repo_data) - self._validate_package_files(package_files) - return self.controller.create_repository(repo_data, package_files) - - def get_packages(self, repos_data, requirements_data=None, - include_mandatory=False, filter_data=None): - """Gets the list of packages from repository(es). - - :param repos_data: The list of repository descriptions - :param requirements_data: The list of package`s requirements - that should be included - :param include_mandatory: if True, all mandatory packages will be - included - :param filter_data: A set of filters that is used to exclude - those packages which match one of filters - :return: the set of packages - """ - repos = self._load_repositories(repos_data) - requirements = self._load_requirements(requirements_data) - exclude_filter = self._load_filter(filter_data) - return self._get_packages(repos, requirements, - include_mandatory, exclude_filter) - - def clone_repositories(self, repos_data, requirements_data, destination, - include_source=False, include_locale=False, - include_mandatory=False, filter_data=None): - """Creates the clones of specified repositories in local folder. - - :param repos_data: The list of repository descriptions - :param requirements_data: The list of package`s requirements - that should be included - :param destination: the destination folder path - :param include_source: if True, the source packages - will be copied as well. - :param include_locale: if True, the locales will be copied as well. - :param include_mandatory: if True, all mandatory packages will be - included - :param filter_data: A set of filters that is used to exclude - those packages which match one of filters - :return: count of copied and total packages. - """ - - repos = self._load_repositories(repos_data) - reqs = self._load_requirements(requirements_data) - exclude_filter = self._load_filter(filter_data) - all_packages = self._get_packages( - repos, reqs, include_mandatory, exclude_filter) - package_groups = defaultdict(set) - for pkg in all_packages: - package_groups[pkg.repository].add(pkg) - - stat = CopyStatistics() - mirrors = defaultdict(set) - # group packages by mirror - for repo, packages in six.iteritems(package_groups): - mirror = self.controller.fork_repository( - repo, destination, include_source, include_locale - ) - mirrors[mirror].update(packages) - - # add new packages to mirrors - for mirror, packages in six.iteritems(mirrors): - self.controller.assign_packages( - mirror, packages, stat.on_package_copied - ) - return stat - - def get_unresolved_dependencies(self, repos_data): - """Gets list of unresolved dependencies for repository(es). - - :param repos_data: The list of repository descriptions - :return: list of unresolved dependencies - """ - packages = PackagesTree() - self._load_packages(self._load_repositories(repos_data), packages.add) - return packages.get_unresolved_dependencies() - - def _get_packages(self, repos, requirements, include_mandatory, - exclude_filter): - if requirements is not None: - forest = PackagesForest() - for repo in repos: - self.controller.load_packages(repo, forest.add_tree().add) - return forest.get_packages(requirements, include_mandatory) - - packages = set() - consumer = packages.add - if exclude_filter is not None: - def consumer(p): - if not exclude_filter(p): - packages.add(p) - self._load_packages(repos, consumer) - return packages - - def _load_packages(self, repos, consumer): - for repo in repos: - self.controller.load_packages(repo, consumer) - - def _load_repositories(self, repos_data): - for repo_data in repos_data: - self._validate_repo_data(repo_data) - return self.controller.load_repositories(repos_data) - - def _load_requirements(self, requirements_data): - if requirements_data is None: - return - - self._validate_requirements_data(requirements_data) - result = [] - for r in requirements_data: - versions = r.get('versions', None) - if versions is None: - result.append(PackageRelation.from_args((r['name'],))) - else: - for version in versions: - result.append(PackageRelation.from_args( - ([r['name']] + version.split(None, 1)) - )) - return result - - def _load_filter(self, filter_data): - """Loads filter from filter data. - - Property value could be a string or a python regexp. - Example of filters data: - - name: full-package-name - section: section1 - - name: /^.*substr/ - - :param filter_data: A list of filters - :return: Lambda that could match a particular package. - """ - - if filter_data is None: - return - - self._validate_filter_data(filter_data) - - def get_pattern_match(pattern, key, value): - return lambda p: pattern.match(getattr(p, key)) - - def get_exact_match(key, value): - return lambda p: getattr(p, key) == value - - def get_logical_and(filters): - return lambda p: all((f(p) for f in filters)) - - def get_logical_or(filters): - return lambda p: any((f(p) for f in filters)) - - filters = [] - for fdata in filter_data: - matchers = [] - for key, value in six.iteritems(fdata): - if value.startswith('/') and value.endswith('/'): - pattern = re.compile(value[1:-1]) - matchers.append(get_pattern_match(pattern, key, value)) - else: - matchers.append(get_exact_match(key, value)) - filters.append(get_logical_and(matchers)) - return get_logical_or(filters) - - def _validate_filter_data(self, filter_data): - self._validate_data(filter_data, PACKAGE_FILTER_SCHEMA) - - def _validate_repo_data(self, repo_data): - schema = self.controller.get_repository_data_schema() - self._validate_data(repo_data, schema) - - def _validate_requirements_data(self, requirements_data): - self._validate_data(requirements_data, PACKAGES_SCHEMA) - - def _validate_package_files(self, package_files): - self._validate_data(package_files, PACKAGE_FILES_SCHEMA) - - def _validate_data(self, data, schema): - """Validate the input data using jsonschema validation. - - :param data: a data to validate represented as a dict - :param schema: a schema to validate represented as a dict; - must be in JSON Schema Draft 4 format. - """ - try: - jsonschema.validate(data, schema) - except jsonschema.ValidationError as e: - self._raise_validation_error( - "data", e.message, e.path - ) - except jsonschema.SchemaError as e: - self._raise_validation_error( - "schema", e.message, e.schema_path - ) - - @staticmethod - def _raise_validation_error(what, details, path): - message = "Invalid {0}: {1}.".format(what, details) - if path: - message += "\nField: [{0}]".format( - "][".join(repr(p) for p in path) - ) - raise ValueError(message) diff --git a/packetary/schemas/package_filter_schema.py b/packetary/api/__init__.py similarity index 70% rename from packetary/schemas/package_filter_schema.py rename to packetary/api/__init__.py index e1b01e8..868dcb3 100644 --- a/packetary/schemas/package_filter_schema.py +++ b/packetary/api/__init__.py @@ -16,18 +16,14 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -PACKAGE_FILTER_SCHEMA = { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "group": { - "type": "string" - } - } - } -} +from packetary.api.context import Configuration +from packetary.api.context import Context +from packetary.api.options import RepositoryCopyOptions +from packetary.api.repositories import RepositoryApi + +__all__ = [ + "Configuration", + "Context", + "RepositoryApi", + "RepositoryCopyOptions", +] diff --git a/packetary/api/context.py b/packetary/api/context.py new file mode 100644 index 0000000..77c4b7f --- /dev/null +++ b/packetary/api/context.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from packetary.library.connections import ConnectionsManager +from packetary.library.executor import AsynchronousSection + + +class Configuration(object): + """The configuration holder.""" + + def __init__(self, http_proxy=None, https_proxy=None, + retries_num=0, retry_interval=0, threads_num=0, + ignore_errors_num=0): + """Initialises. + + :param http_proxy: the url of proxy for connections over http, + no-proxy will be used if it is not specified + :param https_proxy: the url of proxy for connections over https, + no-proxy will be used if it is not specified + :param retries_num: the number of retries on errors + :param retry_interval: the minimal time between retries (in seconds) + :param threads_num: the max number of active threads + :param ignore_errors_num: the number of errors that may occurs + before stop processing + """ + + self.http_proxy = http_proxy + self.https_proxy = https_proxy + self.ignore_errors_num = ignore_errors_num + self.retries_num = retries_num + self.retry_interval = retry_interval + self.threads_num = threads_num + + +class Context(object): + """The infra-objects holder.""" + + def __init__(self, config): + """Initialises. + + :param config: the configuration + """ + self._connection = ConnectionsManager( + proxy=config.http_proxy, + secure_proxy=config.https_proxy, + retries_num=config.retries_num, + retry_interval=config.retry_interval + ) + self._threads_num = config.threads_num + self._ignore_errors_num = config.ignore_errors_num + + @property + def connection(self): + """Gets the connection.""" + return self._connection + + def async_section(self, ignore_errors_num=None): + """Gets the execution scope. + + :param ignore_errors_num: custom value for ignore_errors_num, + the class value is used if omitted. + """ + if ignore_errors_num is None: + ignore_errors_num = self._ignore_errors_num + + return AsynchronousSection(self._threads_num, ignore_errors_num) diff --git a/packetary/api/loaders.py b/packetary/api/loaders.py new file mode 100644 index 0000000..c881505 --- /dev/null +++ b/packetary/api/loaders.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import re + +import six + +from packetary.objects.package_relation import PackageRelation + + +def make_pattern_match(name, value): + return lambda o: re.match(value, getattr(o, name)) + + +def make_exact_match(name, value): + return lambda o: getattr(o, name) == value + + +def make_attr_match(name, value): + if value.startswith('/') and value.endswith('/'): + return make_pattern_match(name, value[1:-1]) + return make_exact_match(name, value) + + +def all_of(operators): + return lambda o: all((x(o) for x in operators)) + + +def any_of(operators): + return lambda o: any((f(o) for f in operators)) + + +def load_filters(data): + """Loads filter from filter data. + + Property value could be a string or a python regexp. + Example of filters data: + - name: full-package-name + section: section1 + - name: /^.*substr/ + + :param data: A list of filters + :return: Lambda that could match a particular package. + """ + return any_of([ + all_of([make_attr_match(n, v) for n, v in six.iteritems(attrs)]) + for attrs in data + ]) + + +def load_package_relations(data, consumer): + """Gets the list PackageRelations from descriptions. + + :param data: the descriptions of package relations + :param consumer: the result consumer + """ + if not data: + return + + for d in data: + versions = d.get('versions', None) + if versions is None: + consumer(PackageRelation.from_args((d['name'],))) + else: + for version in versions: + consumer(PackageRelation.from_args( + ([d['name']] + version.split(None, 1)) + )) + + +def get_packages_traverse(data, consumer): + """Gets the traverse to get all packages from repository as relations. + + :param data: the description of repositories to traverse + :param consumer: the requirements consumer + :return: callable that expects package as argument + """ + if not data: + return lambda _: None + + filters_per_repo = { + d['name']: load_filters(d.get('excludes', ())) + for d in data + } + + def traverse(pkg): + if pkg.repository.name in filters_per_repo: + excludes_filter = filters_per_repo[pkg.repository.name] + if not excludes_filter(pkg): + consumer( + PackageRelation.from_args((pkg.name, '=', pkg.version)) + ) + return traverse diff --git a/packetary/schemas/packages_schema.py b/packetary/api/options.py similarity index 58% rename from packetary/schemas/packages_schema.py rename to packetary/api/options.py index 5b83551..ff9dbbd 100644 --- a/packetary/schemas/packages_schema.py +++ b/packetary/api/options.py @@ -16,27 +16,8 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -PACKAGES_SCHEMA = { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "array", - "items": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string" - }, - "versions": { - "type": "array", - "items": [ - { - "type": "string", - "pattern": "^([<>]=?|=)\s+.+$" - } - ] - } - } - } -} + +class RepositoryCopyOptions(object): + def __init__(self, sources=False, localizations=False): + self.sources = sources + self.localizations = localizations diff --git a/packetary/api/repositories.py b/packetary/api/repositories.py new file mode 100644 index 0000000..94d4ff8 --- /dev/null +++ b/packetary/api/repositories.py @@ -0,0 +1,166 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from collections import defaultdict +import logging + +import six + +from packetary.api.context import Context +from packetary.api.options import RepositoryCopyOptions +from packetary.controllers import RepositoryController +from packetary.library.functions import compose +from packetary import objects +from packetary import schemas + +from packetary.api.loaders import get_packages_traverse +from packetary.api.loaders import load_package_relations +from packetary.api.statistics import CopyStatistics +from packetary.api.validators import declare_schema + + +logger = logging.getLogger(__package__) + + +class RepositoryApi(object): + """Provides high-level API to operate with repositories.""" + + CopyOptions = RepositoryCopyOptions + + def __init__(self, controller): + """Initialises. + + :param controller: the repository controller. + """ + self.controller = controller + + def _get_repository_data_schema(self): + return self.controller.get_repository_data_schema() + + def _get_repositories_data_schema(self): + return { + '$schema': 'http://json-schema.org/draft-04/schema#', + 'type': 'array', + 'items': self._get_repository_data_schema() + } + + @classmethod + def create(cls, config, repotype, repoarch): + """Creates the repository API instance. + + :param config: the configuration + :param repotype: the kind of repository(deb, yum, etc) + :param repoarch: the architecture of repository (x86_64 or i386) + """ + context = config if isinstance(config, Context) else Context(config) + return cls(RepositoryController.load(context, repotype, repoarch)) + + @declare_schema(repo_data=_get_repository_data_schema, + package_files=schemas.PACKAGE_FILES_SCHEMA) + def create_repository(self, repo_data, package_files): + """Create new repository with specified packages. + + :param repo_data: The description of repository + :param package_files: The list of URLs of packages + """ + return self.controller.create_repository(repo_data, package_files) + + @declare_schema(repos_data=_get_repositories_data_schema, + requirements_data=schemas.REQUIREMENTS_SCHEMA) + def get_packages(self, repos_data, requirements_data=None): + """Gets the list of packages from repository(es). + + :param repos_data: The list of repository descriptions + :param requirements_data: The list of package`s requirements + that should be included + :return: the set of packages + """ + repositories = self.controller.load_repositories(repos_data) + return self._get_packages(repositories, requirements_data) + + @declare_schema(repos_data=_get_repositories_data_schema, + requirements_data=schemas.REQUIREMENTS_SCHEMA) + def clone_repositories(self, repos_data, destination, + requirements_data=None, options=None): + """Creates the clones of specified repositories in local folder. + + :param repos_data: The list of repository descriptions + :param requirements_data: The list of package`s requirements + that should be included + :param destination: the destination folder path + :param options: the repository copy options + :return: count of copied and total packages. + """ + + repositories = self.controller.load_repositories(repos_data) + all_packages = self._get_packages(repositories, requirements_data) + package_groups = defaultdict(set) + for pkg in all_packages: + package_groups[pkg.repository].add(pkg) + + stat = CopyStatistics() + mirrors = defaultdict(set) + options = options or self.CopyOptions() + # group packages by mirror + for repo, packages in six.iteritems(package_groups): + m = self.controller.fork_repository(repo, destination, options) + mirrors[m].update(packages) + + # add new packages to mirrors + for m, pkgs in six.iteritems(mirrors): + self.controller.assign_packages(m, pkgs, stat.on_package_copied) + return stat + + @declare_schema(repos_data=_get_repositories_data_schema) + def get_unresolved_dependencies(self, repos_data): + """Gets list of unresolved dependencies for repository(es). + + :param repos_data: The list of repository descriptions + :return: list of unresolved dependencies + """ + packages = objects.PackagesTree() + repositories = self.controller.load_repositories(repos_data) + self._load_packages(repositories, packages.add) + return packages.get_unresolved_dependencies() + + def _get_packages(self, repositories, requirements): + if requirements: + forest = objects.PackagesForest() + package_relations = [] + load_package_relations( + requirements.get('packages'), package_relations.append + ) + packages_traverse = get_packages_traverse( + requirements.get('repositories'), package_relations.append + ) + for repo in repositories: + self.controller.load_packages( + repo, + compose(forest.add_tree().add, packages_traverse) + ) + return forest.get_packages( + package_relations, requirements.get('mandatory', True) + ) + + packages = set() + self._load_packages(repositories, packages.add) + return packages + + def _load_packages(self, repos, consumer): + for repo in repos: + self.controller.load_packages(repo, consumer) diff --git a/packetary/objects/statistics.py b/packetary/api/statistics.py similarity index 100% rename from packetary/objects/statistics.py rename to packetary/api/statistics.py diff --git a/packetary/api/validators.py b/packetary/api/validators.py new file mode 100644 index 0000000..89035ed --- /dev/null +++ b/packetary/api/validators.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import functools +import inspect + +import jsonschema + + +_SENTINEL = object() + + +def _get_default_arguments(func): + try: + signature = inspect.signature(func) + return {p.name: p.default for p in signature.parameters.values()} + except AttributeError: + pass + + args = inspect.getargspec(func) + if args.defaults: + return { + k: v for k, v in zip(reversed(args.args), reversed(args.defaults)) + } + return {} + + +def _get_args_count(func): + try: + return len(inspect.signature(func).parameters) + except AttributeError: + pass + return len(inspect.getargspec(func).args) + + +def _validate_data(data, schema): + """Validate the input data using jsonschema validation. + + :param data: a data to validate represented as a dict + :param schema: a schema to validate represented as a dict; + must be in JSON Schema Draft 4 format. + """ + try: + jsonschema.validate(data, schema) + except jsonschema.ValidationError as e: + _raise_validation_error("data", e.message, e.path) + except jsonschema.SchemaError as e: + _raise_validation_error("schema", e.message, e.schema_path) + + +def _raise_validation_error(what, details, path): + message = "Invalid {0}: {1}.".format(what, details) + if path: + message += "\nField: [{0}]".format( + "][".join(repr(p) for p in path) + ) + raise ValueError(message) + + +def _build_validator(schema): + # check that schema is method of class and expected self argument + if callable(schema) and _get_args_count(schema) > 0: + def validator(self, value): + _validate_data(value, schema(self)) + elif callable(schema): + def validator(_, value): + _validate_data(value, schema()) + else: + def validator(_, value): + return _validate_data(value, schema) + return validator + + +def declare_schema(**schemas): + """Declares data schema for function arguments. + + :param schemas: the mapping, where key is argument name, value is schema + the schema may be callable object or method of class + in this case the wrapped function should be method of + same class + :raises ValueError: if the passed data does not fit declared schema + """ + + def decorator(func): + validators = {k: _build_validator(v) for k, v in schemas.items()} + defaults = _get_default_arguments(func) + + @functools.wraps(func) + def wrapper(*args, **kwargs): + bound_args = inspect.getcallargs(func, *args, **kwargs) + for n, v in bound_args.items(): + if v is not defaults.get(n, _SENTINEL) and n in validators: + validators[n](args and args[0] or None, v) + return func(*args, **kwargs) + return wrapper + return decorator diff --git a/packetary/cli/commands/base.py b/packetary/cli/commands/base.py index 8ebe639..916b9fe 100644 --- a/packetary/cli/commands/base.py +++ b/packetary/cli/commands/base.py @@ -23,7 +23,8 @@ import six from packetary.cli.commands.utils import make_display_attr_getter from packetary.cli.commands.utils import read_from_file -from packetary import RepositoryApi + +from packetary import api @six.add_metaclass(abc.ABCMeta) @@ -66,17 +67,17 @@ class BaseRepoCommand(command.Command): :rtype: object """ return self.take_repo_action( - RepositoryApi.create( + api.RepositoryApi.create( self.app_args, parsed_args.type, parsed_args.arch ), parsed_args ) @abc.abstractmethod - def take_repo_action(self, api, parsed_args): + def take_repo_action(self, repo_api, parsed_args): """Takes action on repository. - :param api: the RepositoryApi instance + :param repo_api: the RepositoryApi instance :param parsed_args: the command-line arguments :return: the action result """ @@ -104,32 +105,13 @@ class PackagesMixin(object): def get_parser(self, prog_name): parser = super(PackagesMixin, self).get_parser(prog_name) parser.add_argument( - "--skip-mandatory", - dest='include_mandatory', - action='store_false', - default=True, - help="Do not copy mandatory packages." - ) - - group = parser.add_mutually_exclusive_group() - - group.add_argument( - "-p", "--packages", + "-R", "--requirements", dest='requirements', type=read_from_file, metavar='FILENAME', help="The path to file with list of packages." "See documentation about format." ) - - group.add_argument( - "-f", "--exclude-filter", - dest='exclude_filter_data', - type=read_from_file, - metavar='FILENAME', - help="The path to file with package exclude filter data." - "See documentation about format." - ) return parser diff --git a/packetary/cli/commands/clone.py b/packetary/cli/commands/clone.py index 4afed94..82974db 100644 --- a/packetary/cli/commands/clone.py +++ b/packetary/cli/commands/clone.py @@ -47,15 +47,15 @@ class CloneCommand(PackagesMixin, RepositoriesMixin, BaseRepoCommand): return parser - def take_repo_action(self, api, parsed_args): - stat = api.clone_repositories( + def take_repo_action(self, repo_api, parsed_args): + stat = repo_api.clone_repositories( parsed_args.repositories, - parsed_args.requirements, parsed_args.destination, - parsed_args.sources, - parsed_args.locales, - parsed_args.include_mandatory, - filter_data=parsed_args.exclude_filter_data, + parsed_args.requirements, + repo_api.CopyOptions( + sources=parsed_args.sources, + localizations=parsed_args.locales, + ) ) self.stdout.write( "Packages copied: {0.copied}/{0.total}.\n".format(stat) diff --git a/packetary/cli/commands/create.py b/packetary/cli/commands/create.py index 28c1a92..f545924 100644 --- a/packetary/cli/commands/create.py +++ b/packetary/cli/commands/create.py @@ -43,8 +43,8 @@ class CreateCommand(BaseRepoCommand): ) return parser - def take_repo_action(self, api, parsed_args): - api.create_repository( + def take_repo_action(self, repo_api, parsed_args): + repo_api.create_repository( parsed_args.repository, parsed_args.package_files ) diff --git a/packetary/cli/commands/packages.py b/packetary/cli/commands/packages.py index 9ee8987..8af6565 100644 --- a/packetary/cli/commands/packages.py +++ b/packetary/cli/commands/packages.py @@ -22,7 +22,8 @@ from packetary.cli.commands.base import RepositoriesMixin class ListOfPackages( - PackagesMixin, RepositoriesMixin, BaseProduceOutputCommand): + PackagesMixin, RepositoriesMixin, BaseProduceOutputCommand): + """Gets the list of packages from repository(es).""" columns = ( @@ -37,12 +38,10 @@ class ListOfPackages( "requires", ) - def take_repo_action(self, api, parsed_args): - return api.get_packages( + def take_repo_action(self, repo_api, parsed_args): + return repo_api.get_packages( parsed_args.repositories, - parsed_args.requirements, - parsed_args.include_mandatory, - filter_data=parsed_args.exclude_filter_data, + parsed_args.requirements ) diff --git a/packetary/cli/commands/unresolved.py b/packetary/cli/commands/unresolved.py index f564024..e4e8926 100644 --- a/packetary/cli/commands/unresolved.py +++ b/packetary/cli/commands/unresolved.py @@ -29,8 +29,8 @@ class ListOfUnresolved(RepositoriesMixin, BaseProduceOutputCommand): "alternative", ) - def take_repo_action(self, api, parsed_args): - return api.get_unresolved_dependencies( + def take_repo_action(self, repo_api, parsed_args): + return repo_api.get_unresolved_dependencies( parsed_args.repositories ) diff --git a/packetary/controllers/repository.py b/packetary/controllers/repository.py index 539df7e..99c2ab1 100644 --- a/packetary/controllers/repository.py +++ b/packetary/controllers/repository.py @@ -84,13 +84,12 @@ class RepositoryController(object): connection = self.context.connection self.driver.get_packages(connection, repository, consumer) - def fork_repository(self, repository, destination, source, locale): + def fork_repository(self, repository, destination, options): """Creates copy of repositories. :param repository: the origin repository :param destination: the target folder - :param source: If True, the source packages will be copied too. - :param locale: If True, the localisation will be copied too. + :param options: The options, see RepositoryCopyOptions :return: the mapping origin to cloned repository. """ new_path = os.path.join( @@ -101,7 +100,7 @@ class RepositoryController(object): ) logger.info("cloning repository '%s' to '%s'", repository, new_path) return self.driver.fork_repository( - self.context.connection, repository, new_path, source, locale + self.context.connection, repository, new_path, options ) def assign_packages(self, repository, packages, observer=None): diff --git a/packetary/drivers/base.py b/packetary/drivers/base.py index dd7c2d9..b456c88 100644 --- a/packetary/drivers/base.py +++ b/packetary/drivers/base.py @@ -55,15 +55,13 @@ class RepositoryDriverBase(object): """ @abc.abstractmethod - def fork_repository(self, connection, repository, destination, - source=False, locale=False): + def fork_repository(self, connection, repository, destination, options): """Creates the new repository with same metadata. :param connection: the connection manager instance :param repository: the source repository :param destination: the destination folder - :param source: copy source files - :param locale: copy localisation + :param options: the options :return: The copy of repository """ diff --git a/packetary/drivers/deb_driver.py b/packetary/drivers/deb_driver.py index 0629f2e..2d4cae8 100644 --- a/packetary/drivers/deb_driver.py +++ b/packetary/drivers/deb_driver.py @@ -197,8 +197,7 @@ class DebRepositoryDriver(RepositoryDriverBase): self.logger.info("saved %d packages in %s", count, repository) self._update_suite_index(repository) - def fork_repository(self, connection, repository, destination, - source=False, locale=False): + def fork_repository(self, connection, repository, destination, options): # TODO(download gpk) # TODO(sources and locales) new_repo = copy.copy(repository) diff --git a/packetary/drivers/rpm_driver.py b/packetary/drivers/rpm_driver.py index 1df7584..3077048 100644 --- a/packetary/drivers/rpm_driver.py +++ b/packetary/drivers/rpm_driver.py @@ -162,8 +162,7 @@ class RpmRepositoryDriver(RepositoryDriverBase): groupstree = self._load_groups(connection, repository) self._rebuild_repository(connection, repository, packages, groupstree) - def fork_repository(self, connection, repository, destination, - source=False, locale=False): + def fork_repository(self, connection, repository, destination, options): # TODO(download gpk) # TODO(sources and locales) new_repo = copy.copy(repository) diff --git a/packetary/library/functions.py b/packetary/library/functions.py new file mode 100644 index 0000000..eef26bb --- /dev/null +++ b/packetary/library/functions.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +def compose(*functions): + """Call all functions with same arguments.""" + + def wrapper(*args, **kwargs): + for f in functions: + f(*args, **kwargs) + return wrapper diff --git a/packetary/schemas/__init__.py b/packetary/schemas/__init__.py index f89036f..9577b4d 100644 --- a/packetary/schemas/__init__.py +++ b/packetary/schemas/__init__.py @@ -18,14 +18,12 @@ from packetary.schemas.deb_repo_schema import DEB_REPO_SCHEMA from packetary.schemas.package_files_schema import PACKAGE_FILES_SCHEMA -from packetary.schemas.package_filter_schema import PACKAGE_FILTER_SCHEMA -from packetary.schemas.packages_schema import PACKAGES_SCHEMA +from packetary.schemas.requirements_schema import REQUIREMENTS_SCHEMA from packetary.schemas.rpm_repo_schema import RPM_REPO_SCHEMA __all__ = [ "DEB_REPO_SCHEMA", "PACKAGE_FILES_SCHEMA", - "PACKAGE_FILTER_SCHEMA", - "PACKAGES_SCHEMA", + "REQUIREMENTS_SCHEMA", "RPM_REPO_SCHEMA", ] diff --git a/packetary/schemas/requirements_schema.py b/packetary/schemas/requirements_schema.py new file mode 100644 index 0000000..e532f75 --- /dev/null +++ b/packetary/schemas/requirements_schema.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +REQUIREMENTS_SCHEMA = { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "anyOf": [ + {"required": ["packages"]}, + {"required": ["repositories"]}, + {"required": ["mandatory"]} + ], + "properties": { + "repositories": { + "type": "array", + "items": { + "type": "object", + "required": ["name"], + "properties": { + "name": { + "type": "string" + }, + "excludes": { + "type": "array", + "items": { + "type": "object", + "patternProperties": { + r"[a-z][\w_]*": { + "type": "string" + } + } + } + } + } + } + }, + "packages": { + "type": "array", + "items": { + "type": "object", + "required": ["name"], + "properties": { + "name": { + "type": "string" + }, + "versions": { + "type": "array", + "items": { + "type": "string", + "pattern": "^([<>]=?|=)\s+.+$" + } + } + } + } + }, + "mandatory": { + "type": "boolean" + }, + } +} diff --git a/packetary/tests/base.py b/packetary/tests/base.py index 7ee48bb..a66173e 100644 --- a/packetary/tests/base.py +++ b/packetary/tests/base.py @@ -26,6 +26,8 @@ class TestCase(unittest.TestCase): """Test case base class for all unit tests.""" + maxDiff = None + def _check_cases(self, assertion, cases, method): for exp, value in cases: assertion( diff --git a/packetary/tests/stubs/helpers.py b/packetary/tests/stubs/helpers.py index 317e923..0b1eb0f 100644 --- a/packetary/tests/stubs/helpers.py +++ b/packetary/tests/stubs/helpers.py @@ -36,11 +36,9 @@ class CallbacksAdapter(mock.MagicMock): else: callback = None - if not callable(callback): - return super(CallbacksAdapter, self).__call__(*args, **kwargs) - - args = args[:-1] data = super(CallbacksAdapter, self).__call__(*args, **kwargs) + if not callable(callback): + return data if isinstance(data, list): for d in data: diff --git a/packetary/tests/test_api_context.py b/packetary/tests/test_api_context.py new file mode 100644 index 0000000..385ff81 --- /dev/null +++ b/packetary/tests/test_api_context.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import mock + +from packetary.api import context + +from packetary.tests import base + + +class TestContext(base.TestCase): + @classmethod + def setUpClass(cls): + cls.config = context.Configuration( + threads_num=2, + ignore_errors_num=3, + retries_num=5, + retry_interval=10, + http_proxy="http://localhost", + https_proxy="https://localhost" + ) + + @mock.patch("packetary.api.context.ConnectionsManager") + def test_initialise_connection_manager(self, conn_manager): + ctx = context.Context(self.config) + conn_manager.assert_called_once_with( + proxy="http://localhost", + secure_proxy="https://localhost", + retries_num=5, + retry_interval=10 + ) + + self.assertIs(conn_manager(), ctx.connection) + + @mock.patch("packetary.api.context.AsynchronousSection") + def test_asynchronous_section(self, async_section): + ctx = context.Context(self.config) + s = ctx.async_section() + async_section.assert_called_with(2, 3) + self.assertIs(s, async_section()) + ctx.async_section(0) + async_section.assert_called_with(2, 0) diff --git a/packetary/tests/test_api_loaders.py b/packetary/tests/test_api_loaders.py new file mode 100644 index 0000000..8000698 --- /dev/null +++ b/packetary/tests/test_api_loaders.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from packetary.api import loaders + +from packetary.tests import base +from packetary.tests.stubs import generator + + +class TestDataLoaders(base.TestCase): + def test_load_filter(self): + filter_data = [ + {"name": "p1", "group": "g1"}, + {"name": "p2"}, + {"group": "g3"}, + {"name": "/^.5/", "group": "/^.*3/"}, + {"group": "/^.*4/"}, + ] + filters = loaders.load_filters(filter_data) + cases = [ + (True, (generator.gen_package(name='p1', group='g1'),)), + (True, (generator.gen_package(name="p2", group="g1"),)), + (False, (generator.gen_package(name="p3", group="g2"),)), + (True, (generator.gen_package(name="p4", group="g3"),)), + (True, (generator.gen_package(name="p5", group="g3"),)), + (True, (generator.gen_package(name="p6", group="g4"),)), + ] + self._check_cases(self.assertIs, cases, filters) + self.assertFalse(loaders.load_filters([])(cases[0][1][0])) + + def test_load_package_relations(self): + data = [ + {'name': 'test1'}, + {'name': 'test2', 'versions': ['> 1', '< 3']}, + ] + expected = [ + str(generator.gen_relation('test1')), + str(generator.gen_relation('test2', ['<', '3'])), + str(generator.gen_relation('test2', ['>', '1'])), + ] + actual = [] + loaders.load_package_relations(data, lambda x: actual.append(str(x))) + self.assertItemsEqual(expected, actual) + actual = [] + loaders.load_package_relations(None, actual.append) + self.assertEqual([], actual) + + def test_get_packages_traverse(self): + data = [{ + 'name': 'r1', + 'excludes': [{'name': 'p1'}] + }] + repo = generator.gen_repository(name='r1') + repo2 = generator.gen_repository(name='r2') + packages = [ + generator.gen_package(name='p1', version=1, repository=repo), + generator.gen_package(name='p2', version=2, repository=repo), + generator.gen_package(name='p3', version=2, repository=repo2), + generator.gen_package(name='p4', version=2, repository=repo2) + ] + actual = [] + traverse = loaders.get_packages_traverse( + data, lambda x: actual.append(str(x)) + ) + for p in packages: + traverse(p) + + expected = [str(generator.gen_relation('p2', ['=', '2']))] + self.assertItemsEqual(expected, actual) diff --git a/packetary/tests/test_api_validators.py b/packetary/tests/test_api_validators.py new file mode 100644 index 0000000..e71b053 --- /dev/null +++ b/packetary/tests/test_api_validators.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import mock + +from jsonschema import SchemaError +from jsonschema import ValidationError + +from packetary.api import validators + +from packetary.tests import base + + +@mock.patch('packetary.api.validators.jsonschema', + ValidationError=ValidationError, SchemaError=SchemaError) +class TestDataValidators(base.TestCase): + @classmethod + def setUpClass(cls): + cls.data = {'key': 'value'} + cls.schema = {'type': 'object'} + + def test_validate_data(self, jsonschema_mock): + validators._validate_data(self.data, self.schema) + jsonschema_mock.validate.assert_called_once_with( + self.data, self.schema + ) + + def test_validate_invalid_data(self, jsonschema_mock): + paths = [(("a", 0), "\['a'\]\[0\]"), ((), "")] + for path, details in paths: + msg = "Invalid data: error." + if details: + msg += "\nField: {0}".format(details) + with self.assertRaisesRegexp(ValueError, msg): + jsonschema_mock.validate.side_effect = ValidationError( + "error", path=path + ) + validators._validate_data(self.data, self.schema) + + msg = "Invalid schema: error." + if details: + msg += "\nField: {0}".format(details) + with self.assertRaisesRegexp(ValueError, msg): + jsonschema_mock.validate.side_effect = SchemaError( + "error", schema_path=path + ) + validators._validate_data(self.data, self.schema) + + def test_build_validator(self, jsonschema_mock): + schemas = [ + lambda this: this.schema, + lambda: self.schema, + self.schema + ] + for schema in schemas: + jsonschema_mock.reset() + validator = validators._build_validator(schema) + validator(self, self.data) + jsonschema_mock.validate.assert_called_with(self.data, self.schema) + + def test_declare_schema_default_does_not_check(self, jsonschema_mock): + func = validators.declare_schema(p=self.schema)(lambda p=None: None) + func(None) + self.assertEqual(0, jsonschema_mock.validate.call_count) + + def test_declare_schema_check_data(self, jsonschema_mock): + func = validators.declare_schema(p=self.schema)(lambda p: None) + func(self.data) + jsonschema_mock.validate.assert_called_with(self.data, self.schema) + + def test_declare_schema_if_schema_is_method(self, jsonschema_mock): + func = validators.declare_schema(p=lambda x: x.schema)( + lambda x, p: None + ) + func(self, self.data) + jsonschema_mock.validate.assert_called_with(self.data, self.schema) diff --git a/packetary/tests/test_cli_commands.py b/packetary/tests/test_cli_commands.py index 0b6e139..b36e128 100644 --- a/packetary/tests/test_cli_commands.py +++ b/packetary/tests/test_cli_commands.py @@ -16,20 +16,22 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import mock import subprocess +import mock + # The cmd2 does not work with python3.5 # because it tries to get access to the property mswindows, # that was removed in 3.5 subprocess.mswindows = False from packetary.api import RepositoryApi +from packetary.api.statistics import CopyStatistics from packetary.cli.commands import clone from packetary.cli.commands import create from packetary.cli.commands import packages from packetary.cli.commands import unresolved -from packetary.objects.statistics import CopyStatistics + from packetary.tests import base from packetary.tests.stubs.generator import gen_package from packetary.tests.stubs.generator import gen_relation @@ -38,7 +40,7 @@ from packetary.tests.stubs.generator import gen_repository @mock.patch("packetary.cli.commands.base.BaseRepoCommand.stdout") @mock.patch("packetary.cli.commands.base.read_from_file") -@mock.patch("packetary.cli.commands.base.RepositoryApi") +@mock.patch("packetary.cli.commands.base.api.RepositoryApi") class TestCliCommands(base.TestCase): common_argv = [ "--ignore-errors-num=3", @@ -51,11 +53,10 @@ class TestCliCommands(base.TestCase): clone_argv = [ "-r", "repositories.yaml", - "-p", "packages.yaml", + "-R", "requirements.yaml", "-d", "/root", "-t", "deb", "-a", "x86_64", - "--skip-mandatory" ] create_argv = [ @@ -95,7 +96,7 @@ class TestCliCommands(base.TestCase): def test_clone_cmd(self, api_mock, read_file_mock, stdout_mock): read_file_mock.side_effect = [ [{"name": "repo"}], - [{"name": "package"}], + {'packages': [{"name": "package"}]}, ] api_instance = self.get_api_instance_mock(api_mock) api_instance.clone_repositories.return_value = CopyStatistics() @@ -105,13 +106,13 @@ class TestCliCommands(base.TestCase): ) self.check_common_config(api_mock.create.call_args[0][0]) read_file_mock.assert_any_call("repositories.yaml") - read_file_mock.assert_any_call("packages.yaml") + read_file_mock.assert_any_call("requirements.yaml") api_instance.clone_repositories.assert_called_once_with( - [{"name": "repo"}], [{"name": "package"}], "/root", - False, - False, - False, - filter_data=None, + [{"name": "repo"}], "/root", {'packages': [{"name": "package"}]}, + api_instance.CopyOptions.return_value + ) + api_instance.CopyOptions.assert_called_once_with( + sources=False, localizations=False, ) stdout_mock.write.assert_called_once_with( "Packages copied: 0/0.\n" @@ -132,7 +133,7 @@ class TestCliCommands(base.TestCase): ) self.check_common_config(api_mock.create.call_args[0][0]) api_instance.get_packages.assert_called_once_with( - [{"name": "repo"}], None, True, filter_data=None + [{"name": "repo"}], None ) self.assertIn( "test1; test1.pkg", diff --git a/packetary/tests/test_deb_driver.py b/packetary/tests/test_deb_driver.py index e58f8f5..3359a9b 100644 --- a/packetary/tests/test_deb_driver.py +++ b/packetary/tests/test_deb_driver.py @@ -230,7 +230,7 @@ class TestDebDriver(base.TestCase): ] open.side_effect = files new_repo = self.driver.fork_repository( - self.connection, self.repo, "/root/test" + self.connection, self.repo, "/root/test", None ) self.assertEqual(self.repo.name, new_repo.name) self.assertEqual(self.repo.architecture, new_repo.architecture) diff --git a/packetary/tests/test_repository_api.py b/packetary/tests/test_repository_api.py index 23ecd0e..5f0231d 100644 --- a/packetary/tests/test_repository_api.py +++ b/packetary/tests/test_repository_api.py @@ -19,62 +19,77 @@ import copy import mock -import jsonschema +from packetary import api +from packetary.controllers.repository import RepositoryController +from packetary import schemas -from packetary.api import Configuration -from packetary.api import Context -from packetary.api import RepositoryApi -from packetary.schemas import PACKAGE_FILES_SCHEMA -from packetary.schemas import PACKAGE_FILTER_SCHEMA -from packetary.schemas import PACKAGES_SCHEMA from packetary.tests import base from packetary.tests.stubs import generator from packetary.tests.stubs.helpers import CallbacksAdapter -@mock.patch("packetary.api.jsonschema") +@mock.patch("packetary.api.validators.jsonschema") class TestRepositoryApi(base.TestCase): def setUp(self): - self.controller = CallbacksAdapter() - self.api = RepositoryApi(self.controller) - self.repo_data = {"name": "repo1", "uri": "file:///repo1"} - self.requirements_data = [ - {"name": "test1"}, {"name": "test2", "versions": ["< 3", "> 1"]} - ] + self.controller = CallbacksAdapter(spec=RepositoryController) + self.api = api.RepositoryApi(self.controller) self.schema = {} - self.repo = generator.gen_repository(**self.repo_data) - self.controller.load_repositories.return_value = [self.repo] self.controller.get_repository_data_schema.return_value = self.schema - self._generate_packages() + + def _generate_repositories(self, count=1): + self.repos_data = [ + {"name": "repo{0}".format(i), "uri": "file:///repo{0}".format(i)} + for i in range(count) + ] + self.repos = [ + generator.gen_repository(**data) for data in self.repos_data + ] + self.controller.load_repositories.return_value = self.repos + + def _generate_mirrors(self): + mirrors = {} + for repo in self.repos: + mirror = copy.copy(repo) + mirror.url = "file:///mirror/{0}".format(repo.name) + mirrors[repo] = mirror + self.controller.fork_repository.side_effect = lambda *x: mirrors[x[0]] + self.mirrors = mirrors def _generate_packages(self): self.packages = [ - generator.gen_package(idx=1, repository=self.repo, requires=None), - generator.gen_package(idx=2, repository=self.repo, requires=None), - generator.gen_package( - idx=3, repository=self.repo, mandatory=True, - requires=[generator.gen_relation("package2")] - ), - generator.gen_package( - idx=4, repository=self.repo, mandatory=False, - requires=[generator.gen_relation("package1")] - ), - generator.gen_package( - idx=5, repository=self.repo, - requires=[generator.gen_relation("package6")]) + [ + generator.gen_package( + name='{0}_1'.format(r.name), repository=r, requires=None + ), + generator.gen_package( + name='{0}_2'.format(r.name), repository=r, requires=None + ), + generator.gen_package( + name='{0}_3'.format(r.name), repository=r, mandatory=True, + requires=[generator.gen_relation("{0}_2".format(r.name))] + ), + generator.gen_package( + name='{0}_4'.format(r.name), repository=r, mandatory=False, + requires=[generator.gen_relation("{0}_1".format(r.name))] + ), + generator.gen_package( + name='{0}_5'.format(r.name), repository=r, + requires=[generator.gen_relation("unresolved")] + ), + ] + for r in self.repos ] - self.controller.load_packages.return_value = self.packages + self.controller.load_packages.side_effect = self.packages - @mock.patch("packetary.api.RepositoryController") - @mock.patch("packetary.api.ConnectionsManager") - def test_create_with_config(self, connection_mock, controller_mock, - jsonschema_mock): - config = Configuration( + @mock.patch("packetary.api.context.ConnectionsManager") + @mock.patch("packetary.api.repositories.RepositoryController") + def test_create_with_config(self, controller_mock, connection_mock, _): + config = api.Configuration( http_proxy="http://localhost", https_proxy="https://localhost", retries_num=10, retry_interval=1, threads_num=8, ignore_errors_num=6 ) - RepositoryApi.create(config, "deb", "x86_64") + api.RepositoryApi.create(config, "deb", "x86_64") connection_mock.assert_called_once_with( proxy="http://localhost", secure_proxy="https://localhost", @@ -85,17 +100,16 @@ class TestRepositoryApi(base.TestCase): mock.ANY, "deb", "x86_64" ) - @mock.patch("packetary.api.RepositoryController") - @mock.patch("packetary.api.ConnectionsManager") - def test_create_with_context(self, connection_mock, controller_mock, - jsonschema_mock): - config = Configuration( + @mock.patch("packetary.api.context.ConnectionsManager") + @mock.patch("packetary.api.repositories.RepositoryController") + def test_create_with_context(self, controller_mock, connection_mock, _): + config = api.Configuration( http_proxy="http://localhost", https_proxy="https://localhost", retries_num=10, retry_interval=1, threads_num=8, ignore_errors_num=6 ) - context = Context(config) - RepositoryApi.create(context, "deb", "x86_64") + context = api.Context(config) + api.RepositoryApi.create(context, "deb", "x86_64") connection_mock.assert_called_once_with( proxy="http://localhost", secure_proxy="https://localhost", @@ -108,299 +122,109 @@ class TestRepositoryApi(base.TestCase): def test_create_repository(self, jsonschema_mock): file_urls = ["file://test1.pkg"] - self.api.create_repository(self.repo_data, file_urls) + self._generate_repositories(1) + self.api.create_repository(self.repos_data[0], file_urls) self.controller.create_repository.assert_called_once_with( - self.repo_data, file_urls - ) - jsonschema_mock.validate.assert_has_calls( - [ - mock.call(self.repo_data, self.schema), - mock.call(file_urls, PACKAGE_FILES_SCHEMA), - ] + self.repos_data[0], file_urls ) + jsonschema_mock.validate.assert_has_calls([ + mock.call(self.repos_data[0], self.schema), + mock.call(file_urls, schemas.PACKAGE_FILES_SCHEMA), + ], any_order=True) def test_get_packages_as_is(self, jsonschema_mock): - packages = self.api.get_packages([self.repo_data], None, False, None) - self.assertEqual(5, len(packages)) - self.assertItemsEqual( - self.packages, - packages - ) + self._generate_repositories(1) + self._generate_packages() + packages = self.api.get_packages(self.repos_data) + self.assertEqual(5, len(self.packages[0])) + self.assertItemsEqual(self.packages[0], packages) jsonschema_mock.validate.assert_called_once_with( - self.repo_data, self.schema + self.repos_data, self.api._get_repositories_data_schema() ) - def test_get_packages_by_requirements_with_mandatory(self, - jsonschema_mock): - requirements = [{"name": "package1"}] - packages = self.api.get_packages( - [self.repo_data], requirements, True, None - ) - self.assertEqual(3, len(packages)) + def test_get_packages_by_requirements(self, jsonschema_mock): + self._generate_repositories(2) + self._generate_packages() + requirements = { + 'packages': [{"name": "repo0_1"}], + 'repositories': [{"name": "repo1"}], + 'mandatory': True + } + packages = self.api.get_packages(self.repos_data, requirements) + expected_packages = self.packages[0][:3] + self.packages[1] self.assertItemsEqual( - ["package1", "package2", "package3"], - (x.name for x in packages) - ) - jsonschema_mock.validate.assert_has_calls( - [ - mock.call(self.repo_data, self.schema), - mock.call(requirements, PACKAGES_SCHEMA), - ] - ) - - def test_get_packages_by_requirements_without_mandatory(self, - jsonschema_mock): - requirements = [{"name": "package4"}] - packages = self.api.get_packages( - [self.repo_data], requirements, False, None - ) - self.assertEqual(2, len(packages)) - self.assertItemsEqual( - ["package1", "package4"], - (x.name for x in packages) - ) - jsonschema_mock.validate.assert_has_calls( - [ - mock.call(self.repo_data, self.schema), - mock.call(requirements, PACKAGES_SCHEMA), - ] + [x.name for x in expected_packages], + [x.name for x in packages] ) + repos_schema = self.api._get_repositories_data_schema() + jsonschema_mock.validate.assert_has_calls([ + mock.call(self.repos_data, repos_schema), + mock.call(requirements, schemas.REQUIREMENTS_SCHEMA) + ], any_order=True) def test_clone_repositories_as_is(self, jsonschema_mock): - # return value is used as statistics - mirror = copy.copy(self.repo) - mirror.url = "file:///mirror/repo" - self.controller.fork_repository.return_value = mirror + self._generate_repositories(1) + self._generate_packages() + self._generate_mirrors() + self.controller.assign_packages.return_value = [0, 1, 1, 1, 0, 6] - stats = self.api.clone_repositories([self.repo_data], None, "/mirror") + options = api.RepositoryCopyOptions() + stats = self.api.clone_repositories( + self.repos_data, "/mirror", options=options) self.controller.fork_repository.assert_called_once_with( - self.repo, '/mirror', False, False + self.repos[0], '/mirror', options ) self.controller.assign_packages.assert_called_once_with( - mirror, set(self.packages) + self.mirrors[self.repos[0]], set(self.packages[0]), mock.ANY ) self.assertEqual(6, stats.total) self.assertEqual(4, stats.copied) jsonschema_mock.validate.assert_called_once_with( - self.repo_data, self.schema + self.repos_data, self.api._get_repositories_data_schema() ) - def test_clone_by_requirements_with_mandatory(self, jsonschema_mock): - # return value is used as statistics - mirror = copy.copy(self.repo) - mirror.url = "file:///mirror/repo" - requirements = [{"name": "package1"}] - self.controller.fork_repository.return_value = mirror - self.controller.assign_packages.return_value = [0, 1, 1] + def test_clone_by_requirements(self, jsonschema_mock): + self._generate_repositories(2) + self._generate_packages() + self._generate_mirrors() + requirements = { + 'packages': [{"name": "repo0_1"}], + 'repositories': [{"name": "repo1"}], + 'mandatory': False + } + self.controller.assign_packages.return_value = [0, 1, 1] * 3 stats = self.api.clone_repositories( - [self.repo_data], requirements, - "/mirror", include_mandatory=True + self.repos_data, "/mirror", requirements ) - packages = {self.packages[0], self.packages[1], self.packages[2]} - self.controller.fork_repository.assert_called_once_with( - self.repo, '/mirror', False, False + self.controller.fork_repository.assert_has_calls( + [mock.call(r, '/mirror', mock.ANY) for r in self.repos], + any_order=True ) - self.controller.assign_packages.assert_called_once_with( - mirror, packages - ) - self.assertEqual(3, stats.total) - self.assertEqual(2, stats.copied) - jsonschema_mock.validate.assert_has_calls( - [ - mock.call(self.repo_data, self.schema), - mock.call(requirements, PACKAGES_SCHEMA), - ] - ) - - def test_clone_by_requirements_without_mandatory(self, - jsonschema_mock): - # return value is used as statistics - mirror = copy.copy(self.repo) - mirror.url = "file:///mirror/repo" - requirements = [{"name": "package4"}] - self.controller.fork_repository.return_value = mirror - self.controller.assign_packages.return_value = [0, 4] - stats = self.api.clone_repositories( - [self.repo_data], requirements, - "/mirror", include_mandatory=False - ) - packages = {self.packages[0], self.packages[3]} - self.controller.fork_repository.assert_called_once_with( - self.repo, '/mirror', False, False - ) - self.controller.assign_packages.assert_called_once_with( - mirror, packages - ) - self.assertEqual(2, stats.total) - self.assertEqual(1, stats.copied) - jsonschema_mock.validate.assert_has_calls( - [ - mock.call(self.repo_data, self.schema), - mock.call(requirements, PACKAGES_SCHEMA), - ] - ) - - def test_clone_with_filter(self, jsonschema_mock): - repos_data = "repos_data" - requirements_data = "requirements_data" - filter_data = "filter_data" - repos = "repos" - requirements = "requirements" - exclude_filter = "exclude_filter" - - self.api._load_repositories = mock.Mock(return_value=repos) - self.api._load_requirements = mock.Mock(return_value=requirements) - self.api._load_filter = mock.Mock(return_value=exclude_filter) - self.api._get_packages = mock.Mock(return_value=set()) - self.api.controller = mock.Mock() - - self.api.clone_repositories(repos_data, requirements_data, - "destination", filter_data=filter_data) - - self.api._load_repositories.assert_called_once_with(repos_data) - self.api._load_requirements.assert_called_once_with(requirements_data) - self.api._load_filter.assert_called_once_with(filter_data) - self.api._get_packages.assert_called_once_with( - repos, requirements, False, exclude_filter) - - def test_get_packages_with_exclude_filter(self, jsonschema_mock): - exclude_filter = lambda p: any([p == "p1", p == "p3"]) - self.api._load_packages = CallbacksAdapter() - self.api._load_packages.return_value = ["p1", "p2", "p3", "p4"] - packages = self.api._get_packages("repos", None, False, exclude_filter) - self.assertSetEqual(packages, set(["p2", "p4"])) - - def test_get_packages_without_exclude_filter(self, jsonschema_mock): - self.api._load_packages = CallbacksAdapter() - self.api._load_packages.return_value = ["p1", "p2"] - packages = self.api._get_packages("repos", None, False, None) - self.assertSetEqual(packages, set(["p1", "p2"])) + self.controller.assign_packages.assert_has_calls([ + mock.call( + self.mirrors[self.repos[0]], + set(self.packages[0][:1]), + mock.ANY + ), + mock.call( + self.mirrors[self.repos[1]], + set(self.packages[1]), + mock.ANY + ) + ], any_order=True) + self.assertEqual(18, stats.total) + self.assertEqual(12, stats.copied) + repos_schema = self.api._get_repositories_data_schema() + jsonschema_mock.validate.assert_has_calls([ + mock.call(self.repos_data, repos_schema), + mock.call(requirements, schemas.REQUIREMENTS_SCHEMA) + ], any_order=True) def test_get_unresolved(self, jsonschema_mock): - unresolved = self.api.get_unresolved_dependencies([self.repo_data]) - self.assertItemsEqual(["package6"], (x.name for x in unresolved)) + self._generate_repositories(1) + self._generate_packages() + unresolved = self.api.get_unresolved_dependencies(self.repos_data) + self.assertItemsEqual(["unresolved"], (x.name for x in unresolved)) jsonschema_mock.validate.assert_called_once_with( - self.repo_data, self.schema + self.repos_data, self.api._get_repositories_data_schema() ) - - def test_load_filter_with_none(self, jsonschema_mock): - self.assertIsNone(self.api._load_filter(None)) - - def test_load_filter(self, jsonschema_mock): - self.api._validate_filter_data = mock.Mock() - filter_data = [ - {"name": "p1", "group": "g1"}, - {"name": "p2"}, - {"group": "g3"}, - {"name": "/^.5/", "group": "/^.*3/"}, - {"group": "/^.*4/"}, - ] - exclude_filter = self.api._load_filter(filter_data) - - p1 = generator.gen_package(name="p1", group="g1") - p2 = generator.gen_package(name="p2", group="g1") - p3 = generator.gen_package(name="p3", group="g2") - p4 = generator.gen_package(name="p4", group="g3") - p5 = generator.gen_package(name="p5", group="g3") - p6 = generator.gen_package(name="p6", group="g4") - - cases = [ - (True, (p1,)), - (True, (p2,)), - (False, (p3,)), - (True, (p4,)), - (True, (p5,)), - (True, (p6,)), - ] - self._check_cases(self.assertEqual, cases, exclude_filter) - - def test_validate_filter_data(self, jsonschema_mock): - self.api._validate_data = mock.Mock() - self.api._validate_filter_data("filter_data") - self.api._validate_data.assert_called_once_with("filter_data", - PACKAGE_FILTER_SCHEMA) - - def test_load_requirements(self, jsonschema_mock): - expected = { - generator.gen_relation("test1"), - generator.gen_relation("test2", ["<", "3"]), - generator.gen_relation("test2", [">", "1"]), - } - actual = set(self.api._load_requirements( - self.requirements_data - )) - self.assertEqual(expected, actual) - self.assertIsNone(self.api._load_requirements(None)) - jsonschema_mock.validate.assert_called_once_with( - self.requirements_data, - PACKAGES_SCHEMA - ) - - def test_validate_data(self, jsonschema_mock): - self.api._validate_data(self.repo_data, self.schema) - jsonschema_mock.validate.assert_called_once_with( - self.repo_data, self.schema - ) - - def test_validate_invalid_data(self, jschema_m): - jschema_m.ValidationError = jsonschema.ValidationError - jschema_m.SchemaError = jsonschema.SchemaError - paths = [(("a", 0), "\['a'\]\[0\]"), ((), "")] - for path, details in paths: - msg = "Invalid data: error." - if details: - msg += "\nField: {0}".format(details) - with self.assertRaisesRegexp(ValueError, msg): - jschema_m.validate.side_effect = jsonschema.ValidationError( - "error", path=path - ) - self.api._validate_data([], {}) - jschema_m.validate.assert_called_with([], {}) - jschema_m.validate.reset_mock() - - msg = "Invalid schema: error." - if details: - msg += "\nField: {0}".format(details) - with self.assertRaisesRegexp(ValueError, msg): - jschema_m.validate.side_effect = jsonschema.SchemaError( - "error", schema_path=path - ) - self.api._validate_data([], {}) - jschema_m.validate.assert_called_with([], {}) - - -class TestContext(base.TestCase): - @classmethod - def setUpClass(cls): - cls.config = Configuration( - threads_num=2, - ignore_errors_num=3, - retries_num=5, - retry_interval=10, - http_proxy="http://localhost", - https_proxy="https://localhost" - ) - - @mock.patch("packetary.api.ConnectionsManager") - def test_initialise_connection_manager(self, conn_manager): - context = Context(self.config) - conn_manager.assert_called_once_with( - proxy="http://localhost", - secure_proxy="https://localhost", - retries_num=5, - retry_interval=10 - ) - - self.assertIs( - conn_manager(), - context.connection - ) - - @mock.patch("packetary.api.AsynchronousSection") - def test_asynchronous_section(self, async_section): - context = Context(self.config) - s = context.async_section() - async_section.assert_called_with(2, 3) - self.assertIs(s, async_section()) - context.async_section(0) - async_section.assert_called_with(2, 0) diff --git a/packetary/tests/test_repository_contoller.py b/packetary/tests/test_repository_contoller.py index e07169c..a407eaa 100644 --- a/packetary/tests/test_repository_contoller.py +++ b/packetary/tests/test_repository_contoller.py @@ -60,7 +60,7 @@ class TestRepositoryController(base.TestCase): repos = self.ctrl.load_repositories([repo_data]) self.driver.get_repository.assert_called_once_with( - self.context.connection, repo_data, self.ctrl.arch + self.context.connection, repo_data, self.ctrl.arch, mock.ANY ) self.assertEqual([repo], repos) @@ -93,14 +93,14 @@ class TestRepositoryController(base.TestCase): clone.url = "/root/repo" self.driver.fork_repository.return_value = clone self.context.connection.retrieve.side_effect = [0, 10] - self.ctrl.fork_repository(repo, "./repo", False, False) + self.ctrl.fork_repository(repo, "./repo", None) self.driver.fork_repository.assert_called_once_with( - self.context.connection, repo, "./repo/test", False, False + self.context.connection, repo, "./repo/test", None ) repo.path = "os" - self.ctrl.fork_repository(repo, "./repo", False, False) + self.ctrl.fork_repository(repo, "./repo", None) self.driver.fork_repository.assert_called_with( - self.context.connection, repo, "./repo/os", False, False + self.context.connection, repo, "./repo/os", None ) def test_copy_packages(self): diff --git a/packetary/tests/test_rpm_driver.py b/packetary/tests/test_rpm_driver.py index 837fa42..c231956 100644 --- a/packetary/tests/test_rpm_driver.py +++ b/packetary/tests/test_rpm_driver.py @@ -265,7 +265,8 @@ class TestRpmDriver(base.TestCase): new_repo = self.driver.fork_repository( self.connection, repo, - "/repo/os/x86_64" + "/repo/os/x86_64", + None ) m_ensure_dir_exists.assert_called_once_with("/repo/os/x86_64") self.assertEqual(repo.name, new_repo.name) @@ -310,7 +311,8 @@ class TestRpmDriver(base.TestCase): self.driver.fork_repository( self.connection, repo, - "/repo/os/x86_64" + "/repo/os/x86_64", + None ) self.assertEqual(tmp_file.name, md_config.groupfile) os_mock.unlink.assert_called_once_with(tmp_file.name) diff --git a/packetary/tests/test_schemas.py b/packetary/tests/test_schemas.py index b650098..d0fde36 100644 --- a/packetary/tests/test_schemas.py +++ b/packetary/tests/test_schemas.py @@ -18,14 +18,14 @@ import jsonschema -from packetary.schemas import DEB_REPO_SCHEMA -from packetary.schemas import PACKAGE_FILES_SCHEMA -from packetary.schemas import PACKAGES_SCHEMA -from packetary.schemas import RPM_REPO_SCHEMA +from packetary import schemas + from packetary.tests import base class TestRepositorySchemaBase(base.TestCase): + schema = None + def check_invalid_name(self): self._check_invalid_type('name') @@ -78,7 +78,7 @@ class TestRepositorySchemaBase(base.TestCase): class TestDebRepoSchema(TestRepositorySchemaBase): def setUp(self): - self.schema = DEB_REPO_SCHEMA + self.schema = schemas.DEB_REPO_SCHEMA def test_valid_repo_data(self): repo_data = { @@ -142,7 +142,7 @@ class TestDebRepoSchema(TestRepositorySchemaBase): class TestRpmRepoSchema(TestRepositorySchemaBase): def setUp(self): - self.schema = RPM_REPO_SCHEMA + self.schema = schemas.RPM_REPO_SCHEMA def test_valid_repo_data(self): repo_data = { @@ -170,63 +170,45 @@ class TestRpmRepoSchema(TestRepositorySchemaBase): self.check_invalid_path() -class TestPackagesSchema(base.TestCase): +class TestRequirementsSchema(base.TestCase): + def setUp(self): - self.schema = PACKAGES_SCHEMA + self.schema = schemas.REQUIREMENTS_SCHEMA def test_valid_requirements_data(self): - requirements_data = [ - {"name": "test1", "versions": [">= 1.1.2", "<= 3"]}, - {"name": "test2", "versions": ["< 3", "> 1", ">= 4"]}, - {"name": "test3", "versions": ["= 3"]}, - {"name": "test4", "versions": ["= 3"]}, - {"name": "test4"} - ] + requirements_data = { + "packages": [ + {"name": "test1", "versions": [">= 1.1.2", "<= 3"]}, + {"name": "test2", "versions": ["< 3", "> 1", ">= 4"]}, + {"name": "test3", "versions": ["= 3"]}, + {"name": "test4", "versions": ["= 3"]}, + {"name": "test4"} + ], + "repositories": [ + {"name": "repo1", "excludes": [{"name": "/a.+/"}]}, + {"name": "repo1", "excludes": [{"group": "debug"}]}, + {"name": "repo1"} + ], + "all_mandatory": True, + "options": ["localizations", "sources"] + } self.assertNotRaises( jsonschema.ValidationError, jsonschema.validate, requirements_data, self.schema ) - def test_validation_fail_for_required_properties(self): - requirements_data = [ - [{"versions": ["< 3", "> 1"]}] + def test_validation_fail_if_missed_required_properties(self): + test_data = [ + {}, + {"packages": [{"version": "=2.0.0"}]}, + {"repositories": [{"excludes": [{"name": "test"}]}]}, ] - for data in requirements_data: - self.assertRaisesRegexp( - jsonschema.ValidationError, - "is a required property", - jsonschema.validate, data, self.schema + for data in test_data: + self.assertRaises( + jsonschema.ValidationError, jsonschema.validate, + data, self.schema ) - def test_validation_fail_if_name_is_invalid(self): - requirements_data = [ - {"name": 123, "versions": [">= 1.1.2", "<= 3"]}, - ] - self.assertRaisesRegexp( - jsonschema.ValidationError, "123 is not of type 'string'", - jsonschema.validate, requirements_data, self.schema - ) - - def test_validation_fail_if_versions_not_array(self): - requirements_data = [ - {"name": "test1", "versions": 123} - ] - self.assertRaisesRegexp( - jsonschema.ValidationError, "123 is not of type 'array'", - jsonschema.validate, requirements_data, - self.schema - ) - - def test_validation_fail_if_versions_not_string(self): - requirements_data = [ - {"name": "test1", "versions": [123]} - ] - self.assertRaisesRegexp( - jsonschema.ValidationError, "123 is not of type 'string'", - jsonschema.validate, requirements_data, - self.schema - ) - def test_validation_fail_if_versions_not_match(self): versions = [ ["1.1.2"], # relational operator @@ -240,14 +222,15 @@ class TestPackagesSchema(base.TestCase): for version in versions: self.assertRaisesRegexp( jsonschema.ValidationError, "does not match", - jsonschema.validate, version, - self.schema['items']['properties']['versions'] + jsonschema.validate, + {"packages": [{"name": "test", "versions": version}]}, + self.schema ) class TestPackageFilesSchema(base.TestCase): def setUp(self): - self.schema = PACKAGE_FILES_SCHEMA + self.schema = schemas.PACKAGE_FILES_SCHEMA def test_valid_file_urls(self): file_urls = [