Introduced new scheme to declare requirements
the requirements can contain the following sections: packages - the list of packages that are needed repositories - the list of repositories, packages from that are needed mandatory - boolean flag that uses to automatically copy mandatory packages Change-Id: Ic26f991c0bf1e9819005cd4bbe7ed40228b2ce1b
This commit is contained in:
parent
4569ca760f
commit
d661055322
@ -18,19 +18,13 @@
|
||||
|
||||
import pbr.version
|
||||
|
||||
from packetary.api import Configuration
|
||||
from packetary.api import Context
|
||||
from packetary.api import RepositoryApi
|
||||
from packetary import api
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Configuration",
|
||||
"Context",
|
||||
"RepositoryApi",
|
||||
]
|
||||
__all__ = ["api", "__version__"]
|
||||
|
||||
try:
|
||||
__version__ = pbr.version.VersionInfo(
|
||||
'packetary').version_string()
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
__version__ = "0.0.0"
|
||||
|
328
packetary/api.py
328
packetary/api.py
@ -1,328 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
from collections import defaultdict
|
||||
import logging
|
||||
import re
|
||||
|
||||
import jsonschema
|
||||
import six
|
||||
|
||||
from packetary.controllers import RepositoryController
|
||||
from packetary.library.connections import ConnectionsManager
|
||||
from packetary.library.executor import AsynchronousSection
|
||||
from packetary.objects import PackageRelation
|
||||
from packetary.objects import PackagesForest
|
||||
from packetary.objects import PackagesTree
|
||||
from packetary.objects.statistics import CopyStatistics
|
||||
from packetary.schemas import PACKAGE_FILES_SCHEMA
|
||||
from packetary.schemas import PACKAGE_FILTER_SCHEMA
|
||||
from packetary.schemas import PACKAGES_SCHEMA
|
||||
|
||||
logger = logging.getLogger(__package__)
|
||||
|
||||
|
||||
class Configuration(object):
|
||||
"""The configuration holder."""
|
||||
|
||||
def __init__(self, http_proxy=None, https_proxy=None,
|
||||
retries_num=0, retry_interval=0, threads_num=0,
|
||||
ignore_errors_num=0):
|
||||
"""Initialises.
|
||||
|
||||
:param http_proxy: the url of proxy for connections over http,
|
||||
no-proxy will be used if it is not specified
|
||||
:param https_proxy: the url of proxy for connections over https,
|
||||
no-proxy will be used if it is not specified
|
||||
:param retries_num: the number of retries on errors
|
||||
:param retry_interval: the minimal time between retries (in seconds)
|
||||
:param threads_num: the max number of active threads
|
||||
:param ignore_errors_num: the number of errors that may occurs
|
||||
before stop processing
|
||||
"""
|
||||
|
||||
self.http_proxy = http_proxy
|
||||
self.https_proxy = https_proxy
|
||||
self.ignore_errors_num = ignore_errors_num
|
||||
self.retries_num = retries_num
|
||||
self.retry_interval = retry_interval
|
||||
self.threads_num = threads_num
|
||||
|
||||
|
||||
class Context(object):
|
||||
"""The infra-objects holder."""
|
||||
|
||||
def __init__(self, config):
|
||||
"""Initialises.
|
||||
|
||||
:param config: the configuration
|
||||
"""
|
||||
self._connection = ConnectionsManager(
|
||||
proxy=config.http_proxy,
|
||||
secure_proxy=config.https_proxy,
|
||||
retries_num=config.retries_num,
|
||||
retry_interval=config.retry_interval
|
||||
)
|
||||
self._threads_num = config.threads_num
|
||||
self._ignore_errors_num = config.ignore_errors_num
|
||||
|
||||
@property
|
||||
def connection(self):
|
||||
"""Gets the connection."""
|
||||
return self._connection
|
||||
|
||||
def async_section(self, ignore_errors_num=None):
|
||||
"""Gets the execution scope.
|
||||
|
||||
:param ignore_errors_num: custom value for ignore_errors_num,
|
||||
the class value is used if omitted.
|
||||
"""
|
||||
if ignore_errors_num is None:
|
||||
ignore_errors_num = self._ignore_errors_num
|
||||
|
||||
return AsynchronousSection(self._threads_num, ignore_errors_num)
|
||||
|
||||
|
||||
class RepositoryApi(object):
|
||||
"""Provides high-level API to operate with repositories."""
|
||||
|
||||
def __init__(self, controller):
|
||||
"""Initialises.
|
||||
|
||||
:param controller: the repository controller.
|
||||
"""
|
||||
self.controller = controller
|
||||
|
||||
@classmethod
|
||||
def create(cls, config, repotype, repoarch):
|
||||
"""Creates the repository API instance.
|
||||
|
||||
:param config: the configuration
|
||||
:param repotype: the kind of repository(deb, yum, etc)
|
||||
:param repoarch: the architecture of repository (x86_64 or i386)
|
||||
"""
|
||||
context = config if isinstance(config, Context) else Context(config)
|
||||
return cls(RepositoryController.load(context, repotype, repoarch))
|
||||
|
||||
def create_repository(self, repo_data, package_files):
|
||||
"""Create new repository with specified packages.
|
||||
|
||||
:param repo_data: The description of repository
|
||||
:param package_files: The list of URLs of packages
|
||||
"""
|
||||
self._validate_repo_data(repo_data)
|
||||
self._validate_package_files(package_files)
|
||||
return self.controller.create_repository(repo_data, package_files)
|
||||
|
||||
def get_packages(self, repos_data, requirements_data=None,
|
||||
include_mandatory=False, filter_data=None):
|
||||
"""Gets the list of packages from repository(es).
|
||||
|
||||
:param repos_data: The list of repository descriptions
|
||||
:param requirements_data: The list of package`s requirements
|
||||
that should be included
|
||||
:param include_mandatory: if True, all mandatory packages will be
|
||||
included
|
||||
:param filter_data: A set of filters that is used to exclude
|
||||
those packages which match one of filters
|
||||
:return: the set of packages
|
||||
"""
|
||||
repos = self._load_repositories(repos_data)
|
||||
requirements = self._load_requirements(requirements_data)
|
||||
exclude_filter = self._load_filter(filter_data)
|
||||
return self._get_packages(repos, requirements,
|
||||
include_mandatory, exclude_filter)
|
||||
|
||||
def clone_repositories(self, repos_data, requirements_data, destination,
|
||||
include_source=False, include_locale=False,
|
||||
include_mandatory=False, filter_data=None):
|
||||
"""Creates the clones of specified repositories in local folder.
|
||||
|
||||
:param repos_data: The list of repository descriptions
|
||||
:param requirements_data: The list of package`s requirements
|
||||
that should be included
|
||||
:param destination: the destination folder path
|
||||
:param include_source: if True, the source packages
|
||||
will be copied as well.
|
||||
:param include_locale: if True, the locales will be copied as well.
|
||||
:param include_mandatory: if True, all mandatory packages will be
|
||||
included
|
||||
:param filter_data: A set of filters that is used to exclude
|
||||
those packages which match one of filters
|
||||
:return: count of copied and total packages.
|
||||
"""
|
||||
|
||||
repos = self._load_repositories(repos_data)
|
||||
reqs = self._load_requirements(requirements_data)
|
||||
exclude_filter = self._load_filter(filter_data)
|
||||
all_packages = self._get_packages(
|
||||
repos, reqs, include_mandatory, exclude_filter)
|
||||
package_groups = defaultdict(set)
|
||||
for pkg in all_packages:
|
||||
package_groups[pkg.repository].add(pkg)
|
||||
|
||||
stat = CopyStatistics()
|
||||
mirrors = defaultdict(set)
|
||||
# group packages by mirror
|
||||
for repo, packages in six.iteritems(package_groups):
|
||||
mirror = self.controller.fork_repository(
|
||||
repo, destination, include_source, include_locale
|
||||
)
|
||||
mirrors[mirror].update(packages)
|
||||
|
||||
# add new packages to mirrors
|
||||
for mirror, packages in six.iteritems(mirrors):
|
||||
self.controller.assign_packages(
|
||||
mirror, packages, stat.on_package_copied
|
||||
)
|
||||
return stat
|
||||
|
||||
def get_unresolved_dependencies(self, repos_data):
|
||||
"""Gets list of unresolved dependencies for repository(es).
|
||||
|
||||
:param repos_data: The list of repository descriptions
|
||||
:return: list of unresolved dependencies
|
||||
"""
|
||||
packages = PackagesTree()
|
||||
self._load_packages(self._load_repositories(repos_data), packages.add)
|
||||
return packages.get_unresolved_dependencies()
|
||||
|
||||
def _get_packages(self, repos, requirements, include_mandatory,
|
||||
exclude_filter):
|
||||
if requirements is not None:
|
||||
forest = PackagesForest()
|
||||
for repo in repos:
|
||||
self.controller.load_packages(repo, forest.add_tree().add)
|
||||
return forest.get_packages(requirements, include_mandatory)
|
||||
|
||||
packages = set()
|
||||
consumer = packages.add
|
||||
if exclude_filter is not None:
|
||||
def consumer(p):
|
||||
if not exclude_filter(p):
|
||||
packages.add(p)
|
||||
self._load_packages(repos, consumer)
|
||||
return packages
|
||||
|
||||
def _load_packages(self, repos, consumer):
|
||||
for repo in repos:
|
||||
self.controller.load_packages(repo, consumer)
|
||||
|
||||
def _load_repositories(self, repos_data):
|
||||
for repo_data in repos_data:
|
||||
self._validate_repo_data(repo_data)
|
||||
return self.controller.load_repositories(repos_data)
|
||||
|
||||
def _load_requirements(self, requirements_data):
|
||||
if requirements_data is None:
|
||||
return
|
||||
|
||||
self._validate_requirements_data(requirements_data)
|
||||
result = []
|
||||
for r in requirements_data:
|
||||
versions = r.get('versions', None)
|
||||
if versions is None:
|
||||
result.append(PackageRelation.from_args((r['name'],)))
|
||||
else:
|
||||
for version in versions:
|
||||
result.append(PackageRelation.from_args(
|
||||
([r['name']] + version.split(None, 1))
|
||||
))
|
||||
return result
|
||||
|
||||
def _load_filter(self, filter_data):
|
||||
"""Loads filter from filter data.
|
||||
|
||||
Property value could be a string or a python regexp.
|
||||
Example of filters data:
|
||||
- name: full-package-name
|
||||
section: section1
|
||||
- name: /^.*substr/
|
||||
|
||||
:param filter_data: A list of filters
|
||||
:return: Lambda that could match a particular package.
|
||||
"""
|
||||
|
||||
if filter_data is None:
|
||||
return
|
||||
|
||||
self._validate_filter_data(filter_data)
|
||||
|
||||
def get_pattern_match(pattern, key, value):
|
||||
return lambda p: pattern.match(getattr(p, key))
|
||||
|
||||
def get_exact_match(key, value):
|
||||
return lambda p: getattr(p, key) == value
|
||||
|
||||
def get_logical_and(filters):
|
||||
return lambda p: all((f(p) for f in filters))
|
||||
|
||||
def get_logical_or(filters):
|
||||
return lambda p: any((f(p) for f in filters))
|
||||
|
||||
filters = []
|
||||
for fdata in filter_data:
|
||||
matchers = []
|
||||
for key, value in six.iteritems(fdata):
|
||||
if value.startswith('/') and value.endswith('/'):
|
||||
pattern = re.compile(value[1:-1])
|
||||
matchers.append(get_pattern_match(pattern, key, value))
|
||||
else:
|
||||
matchers.append(get_exact_match(key, value))
|
||||
filters.append(get_logical_and(matchers))
|
||||
return get_logical_or(filters)
|
||||
|
||||
def _validate_filter_data(self, filter_data):
|
||||
self._validate_data(filter_data, PACKAGE_FILTER_SCHEMA)
|
||||
|
||||
def _validate_repo_data(self, repo_data):
|
||||
schema = self.controller.get_repository_data_schema()
|
||||
self._validate_data(repo_data, schema)
|
||||
|
||||
def _validate_requirements_data(self, requirements_data):
|
||||
self._validate_data(requirements_data, PACKAGES_SCHEMA)
|
||||
|
||||
def _validate_package_files(self, package_files):
|
||||
self._validate_data(package_files, PACKAGE_FILES_SCHEMA)
|
||||
|
||||
def _validate_data(self, data, schema):
|
||||
"""Validate the input data using jsonschema validation.
|
||||
|
||||
:param data: a data to validate represented as a dict
|
||||
:param schema: a schema to validate represented as a dict;
|
||||
must be in JSON Schema Draft 4 format.
|
||||
"""
|
||||
try:
|
||||
jsonschema.validate(data, schema)
|
||||
except jsonschema.ValidationError as e:
|
||||
self._raise_validation_error(
|
||||
"data", e.message, e.path
|
||||
)
|
||||
except jsonschema.SchemaError as e:
|
||||
self._raise_validation_error(
|
||||
"schema", e.message, e.schema_path
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _raise_validation_error(what, details, path):
|
||||
message = "Invalid {0}: {1}.".format(what, details)
|
||||
if path:
|
||||
message += "\nField: [{0}]".format(
|
||||
"][".join(repr(p) for p in path)
|
||||
)
|
||||
raise ValueError(message)
|
@ -16,18 +16,14 @@
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
PACKAGE_FILTER_SCHEMA = {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"group": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
from packetary.api.context import Configuration
|
||||
from packetary.api.context import Context
|
||||
from packetary.api.options import RepositoryCopyOptions
|
||||
from packetary.api.repositories import RepositoryApi
|
||||
|
||||
__all__ = [
|
||||
"Configuration",
|
||||
"Context",
|
||||
"RepositoryApi",
|
||||
"RepositoryCopyOptions",
|
||||
]
|
81
packetary/api/context.py
Normal file
81
packetary/api/context.py
Normal file
@ -0,0 +1,81 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
from packetary.library.connections import ConnectionsManager
|
||||
from packetary.library.executor import AsynchronousSection
|
||||
|
||||
|
||||
class Configuration(object):
|
||||
"""The configuration holder."""
|
||||
|
||||
def __init__(self, http_proxy=None, https_proxy=None,
|
||||
retries_num=0, retry_interval=0, threads_num=0,
|
||||
ignore_errors_num=0):
|
||||
"""Initialises.
|
||||
|
||||
:param http_proxy: the url of proxy for connections over http,
|
||||
no-proxy will be used if it is not specified
|
||||
:param https_proxy: the url of proxy for connections over https,
|
||||
no-proxy will be used if it is not specified
|
||||
:param retries_num: the number of retries on errors
|
||||
:param retry_interval: the minimal time between retries (in seconds)
|
||||
:param threads_num: the max number of active threads
|
||||
:param ignore_errors_num: the number of errors that may occurs
|
||||
before stop processing
|
||||
"""
|
||||
|
||||
self.http_proxy = http_proxy
|
||||
self.https_proxy = https_proxy
|
||||
self.ignore_errors_num = ignore_errors_num
|
||||
self.retries_num = retries_num
|
||||
self.retry_interval = retry_interval
|
||||
self.threads_num = threads_num
|
||||
|
||||
|
||||
class Context(object):
|
||||
"""The infra-objects holder."""
|
||||
|
||||
def __init__(self, config):
|
||||
"""Initialises.
|
||||
|
||||
:param config: the configuration
|
||||
"""
|
||||
self._connection = ConnectionsManager(
|
||||
proxy=config.http_proxy,
|
||||
secure_proxy=config.https_proxy,
|
||||
retries_num=config.retries_num,
|
||||
retry_interval=config.retry_interval
|
||||
)
|
||||
self._threads_num = config.threads_num
|
||||
self._ignore_errors_num = config.ignore_errors_num
|
||||
|
||||
@property
|
||||
def connection(self):
|
||||
"""Gets the connection."""
|
||||
return self._connection
|
||||
|
||||
def async_section(self, ignore_errors_num=None):
|
||||
"""Gets the execution scope.
|
||||
|
||||
:param ignore_errors_num: custom value for ignore_errors_num,
|
||||
the class value is used if omitted.
|
||||
"""
|
||||
if ignore_errors_num is None:
|
||||
ignore_errors_num = self._ignore_errors_num
|
||||
|
||||
return AsynchronousSection(self._threads_num, ignore_errors_num)
|
108
packetary/api/loaders.py
Normal file
108
packetary/api/loaders.py
Normal file
@ -0,0 +1,108 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
import re
|
||||
|
||||
import six
|
||||
|
||||
from packetary.objects.package_relation import PackageRelation
|
||||
|
||||
|
||||
def make_pattern_match(name, value):
|
||||
return lambda o: re.match(value, getattr(o, name))
|
||||
|
||||
|
||||
def make_exact_match(name, value):
|
||||
return lambda o: getattr(o, name) == value
|
||||
|
||||
|
||||
def make_attr_match(name, value):
|
||||
if value.startswith('/') and value.endswith('/'):
|
||||
return make_pattern_match(name, value[1:-1])
|
||||
return make_exact_match(name, value)
|
||||
|
||||
|
||||
def all_of(operators):
|
||||
return lambda o: all((x(o) for x in operators))
|
||||
|
||||
|
||||
def any_of(operators):
|
||||
return lambda o: any((f(o) for f in operators))
|
||||
|
||||
|
||||
def load_filters(data):
|
||||
"""Loads filter from filter data.
|
||||
|
||||
Property value could be a string or a python regexp.
|
||||
Example of filters data:
|
||||
- name: full-package-name
|
||||
section: section1
|
||||
- name: /^.*substr/
|
||||
|
||||
:param data: A list of filters
|
||||
:return: Lambda that could match a particular package.
|
||||
"""
|
||||
return any_of([
|
||||
all_of([make_attr_match(n, v) for n, v in six.iteritems(attrs)])
|
||||
for attrs in data
|
||||
])
|
||||
|
||||
|
||||
def load_package_relations(data, consumer):
|
||||
"""Gets the list PackageRelations from descriptions.
|
||||
|
||||
:param data: the descriptions of package relations
|
||||
:param consumer: the result consumer
|
||||
"""
|
||||
if not data:
|
||||
return
|
||||
|
||||
for d in data:
|
||||
versions = d.get('versions', None)
|
||||
if versions is None:
|
||||
consumer(PackageRelation.from_args((d['name'],)))
|
||||
else:
|
||||
for version in versions:
|
||||
consumer(PackageRelation.from_args(
|
||||
([d['name']] + version.split(None, 1))
|
||||
))
|
||||
|
||||
|
||||
def get_packages_traverse(data, consumer):
|
||||
"""Gets the traverse to get all packages from repository as relations.
|
||||
|
||||
:param data: the description of repositories to traverse
|
||||
:param consumer: the requirements consumer
|
||||
:return: callable that expects package as argument
|
||||
"""
|
||||
if not data:
|
||||
return lambda _: None
|
||||
|
||||
filters_per_repo = {
|
||||
d['name']: load_filters(d.get('excludes', ()))
|
||||
for d in data
|
||||
}
|
||||
|
||||
def traverse(pkg):
|
||||
if pkg.repository.name in filters_per_repo:
|
||||
excludes_filter = filters_per_repo[pkg.repository.name]
|
||||
if not excludes_filter(pkg):
|
||||
consumer(
|
||||
PackageRelation.from_args((pkg.name, '=', pkg.version))
|
||||
)
|
||||
return traverse
|
@ -16,27 +16,8 @@
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
PACKAGES_SCHEMA = {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"name"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"versions": {
|
||||
"type": "array",
|
||||
"items": [
|
||||
{
|
||||
"type": "string",
|
||||
"pattern": "^([<>]=?|=)\s+.+$"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class RepositoryCopyOptions(object):
|
||||
def __init__(self, sources=False, localizations=False):
|
||||
self.sources = sources
|
||||
self.localizations = localizations
|
166
packetary/api/repositories.py
Normal file
166
packetary/api/repositories.py
Normal file
@ -0,0 +1,166 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
from collections import defaultdict
|
||||
import logging
|
||||
|
||||
import six
|
||||
|
||||
from packetary.api.context import Context
|
||||
from packetary.api.options import RepositoryCopyOptions
|
||||
from packetary.controllers import RepositoryController
|
||||
from packetary.library.functions import compose
|
||||
from packetary import objects
|
||||
from packetary import schemas
|
||||
|
||||
from packetary.api.loaders import get_packages_traverse
|
||||
from packetary.api.loaders import load_package_relations
|
||||
from packetary.api.statistics import CopyStatistics
|
||||
from packetary.api.validators import declare_schema
|
||||
|
||||
|
||||
logger = logging.getLogger(__package__)
|
||||
|
||||
|
||||
class RepositoryApi(object):
|
||||
"""Provides high-level API to operate with repositories."""
|
||||
|
||||
CopyOptions = RepositoryCopyOptions
|
||||
|
||||
def __init__(self, controller):
|
||||
"""Initialises.
|
||||
|
||||
:param controller: the repository controller.
|
||||
"""
|
||||
self.controller = controller
|
||||
|
||||
def _get_repository_data_schema(self):
|
||||
return self.controller.get_repository_data_schema()
|
||||
|
||||
def _get_repositories_data_schema(self):
|
||||
return {
|
||||
'$schema': 'http://json-schema.org/draft-04/schema#',
|
||||
'type': 'array',
|
||||
'items': self._get_repository_data_schema()
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def create(cls, config, repotype, repoarch):
|
||||
"""Creates the repository API instance.
|
||||
|
||||
:param config: the configuration
|
||||
:param repotype: the kind of repository(deb, yum, etc)
|
||||
:param repoarch: the architecture of repository (x86_64 or i386)
|
||||
"""
|
||||
context = config if isinstance(config, Context) else Context(config)
|
||||
return cls(RepositoryController.load(context, repotype, repoarch))
|
||||
|
||||
@declare_schema(repo_data=_get_repository_data_schema,
|
||||
package_files=schemas.PACKAGE_FILES_SCHEMA)
|
||||
def create_repository(self, repo_data, package_files):
|
||||
"""Create new repository with specified packages.
|
||||
|
||||
:param repo_data: The description of repository
|
||||
:param package_files: The list of URLs of packages
|
||||
"""
|
||||
return self.controller.create_repository(repo_data, package_files)
|
||||
|
||||
@declare_schema(repos_data=_get_repositories_data_schema,
|
||||
requirements_data=schemas.REQUIREMENTS_SCHEMA)
|
||||
def get_packages(self, repos_data, requirements_data=None):
|
||||
"""Gets the list of packages from repository(es).
|
||||
|
||||
:param repos_data: The list of repository descriptions
|
||||
:param requirements_data: The list of package`s requirements
|
||||
that should be included
|
||||
:return: the set of packages
|
||||
"""
|
||||
repositories = self.controller.load_repositories(repos_data)
|
||||
return self._get_packages(repositories, requirements_data)
|
||||
|
||||
@declare_schema(repos_data=_get_repositories_data_schema,
|
||||
requirements_data=schemas.REQUIREMENTS_SCHEMA)
|
||||
def clone_repositories(self, repos_data, destination,
|
||||
requirements_data=None, options=None):
|
||||
"""Creates the clones of specified repositories in local folder.
|
||||
|
||||
:param repos_data: The list of repository descriptions
|
||||
:param requirements_data: The list of package`s requirements
|
||||
that should be included
|
||||
:param destination: the destination folder path
|
||||
:param options: the repository copy options
|
||||
:return: count of copied and total packages.
|
||||
"""
|
||||
|
||||
repositories = self.controller.load_repositories(repos_data)
|
||||
all_packages = self._get_packages(repositories, requirements_data)
|
||||
package_groups = defaultdict(set)
|
||||
for pkg in all_packages:
|
||||
package_groups[pkg.repository].add(pkg)
|
||||
|
||||
stat = CopyStatistics()
|
||||
mirrors = defaultdict(set)
|
||||
options = options or self.CopyOptions()
|
||||
# group packages by mirror
|
||||
for repo, packages in six.iteritems(package_groups):
|
||||
m = self.controller.fork_repository(repo, destination, options)
|
||||
mirrors[m].update(packages)
|
||||
|
||||
# add new packages to mirrors
|
||||
for m, pkgs in six.iteritems(mirrors):
|
||||
self.controller.assign_packages(m, pkgs, stat.on_package_copied)
|
||||
return stat
|
||||
|
||||
@declare_schema(repos_data=_get_repositories_data_schema)
|
||||
def get_unresolved_dependencies(self, repos_data):
|
||||
"""Gets list of unresolved dependencies for repository(es).
|
||||
|
||||
:param repos_data: The list of repository descriptions
|
||||
:return: list of unresolved dependencies
|
||||
"""
|
||||
packages = objects.PackagesTree()
|
||||
repositories = self.controller.load_repositories(repos_data)
|
||||
self._load_packages(repositories, packages.add)
|
||||
return packages.get_unresolved_dependencies()
|
||||
|
||||
def _get_packages(self, repositories, requirements):
|
||||
if requirements:
|
||||
forest = objects.PackagesForest()
|
||||
package_relations = []
|
||||
load_package_relations(
|
||||
requirements.get('packages'), package_relations.append
|
||||
)
|
||||
packages_traverse = get_packages_traverse(
|
||||
requirements.get('repositories'), package_relations.append
|
||||
)
|
||||
for repo in repositories:
|
||||
self.controller.load_packages(
|
||||
repo,
|
||||
compose(forest.add_tree().add, packages_traverse)
|
||||
)
|
||||
return forest.get_packages(
|
||||
package_relations, requirements.get('mandatory', True)
|
||||
)
|
||||
|
||||
packages = set()
|
||||
self._load_packages(repositories, packages.add)
|
||||
return packages
|
||||
|
||||
def _load_packages(self, repos, consumer):
|
||||
for repo in repos:
|
||||
self.controller.load_packages(repo, consumer)
|
111
packetary/api/validators.py
Normal file
111
packetary/api/validators.py
Normal file
@ -0,0 +1,111 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
import functools
|
||||
import inspect
|
||||
|
||||
import jsonschema
|
||||
|
||||
|
||||
_SENTINEL = object()
|
||||
|
||||
|
||||
def _get_default_arguments(func):
|
||||
try:
|
||||
signature = inspect.signature(func)
|
||||
return {p.name: p.default for p in signature.parameters.values()}
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
args = inspect.getargspec(func)
|
||||
if args.defaults:
|
||||
return {
|
||||
k: v for k, v in zip(reversed(args.args), reversed(args.defaults))
|
||||
}
|
||||
return {}
|
||||
|
||||
|
||||
def _get_args_count(func):
|
||||
try:
|
||||
return len(inspect.signature(func).parameters)
|
||||
except AttributeError:
|
||||
pass
|
||||
return len(inspect.getargspec(func).args)
|
||||
|
||||
|
||||
def _validate_data(data, schema):
|
||||
"""Validate the input data using jsonschema validation.
|
||||
|
||||
:param data: a data to validate represented as a dict
|
||||
:param schema: a schema to validate represented as a dict;
|
||||
must be in JSON Schema Draft 4 format.
|
||||
"""
|
||||
try:
|
||||
jsonschema.validate(data, schema)
|
||||
except jsonschema.ValidationError as e:
|
||||
_raise_validation_error("data", e.message, e.path)
|
||||
except jsonschema.SchemaError as e:
|
||||
_raise_validation_error("schema", e.message, e.schema_path)
|
||||
|
||||
|
||||
def _raise_validation_error(what, details, path):
|
||||
message = "Invalid {0}: {1}.".format(what, details)
|
||||
if path:
|
||||
message += "\nField: [{0}]".format(
|
||||
"][".join(repr(p) for p in path)
|
||||
)
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def _build_validator(schema):
|
||||
# check that schema is method of class and expected self argument
|
||||
if callable(schema) and _get_args_count(schema) > 0:
|
||||
def validator(self, value):
|
||||
_validate_data(value, schema(self))
|
||||
elif callable(schema):
|
||||
def validator(_, value):
|
||||
_validate_data(value, schema())
|
||||
else:
|
||||
def validator(_, value):
|
||||
return _validate_data(value, schema)
|
||||
return validator
|
||||
|
||||
|
||||
def declare_schema(**schemas):
|
||||
"""Declares data schema for function arguments.
|
||||
|
||||
:param schemas: the mapping, where key is argument name, value is schema
|
||||
the schema may be callable object or method of class
|
||||
in this case the wrapped function should be method of
|
||||
same class
|
||||
:raises ValueError: if the passed data does not fit declared schema
|
||||
"""
|
||||
|
||||
def decorator(func):
|
||||
validators = {k: _build_validator(v) for k, v in schemas.items()}
|
||||
defaults = _get_default_arguments(func)
|
||||
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
bound_args = inspect.getcallargs(func, *args, **kwargs)
|
||||
for n, v in bound_args.items():
|
||||
if v is not defaults.get(n, _SENTINEL) and n in validators:
|
||||
validators[n](args and args[0] or None, v)
|
||||
return func(*args, **kwargs)
|
||||
return wrapper
|
||||
return decorator
|
@ -23,7 +23,8 @@ import six
|
||||
|
||||
from packetary.cli.commands.utils import make_display_attr_getter
|
||||
from packetary.cli.commands.utils import read_from_file
|
||||
from packetary import RepositoryApi
|
||||
|
||||
from packetary import api
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
@ -66,17 +67,17 @@ class BaseRepoCommand(command.Command):
|
||||
:rtype: object
|
||||
"""
|
||||
return self.take_repo_action(
|
||||
RepositoryApi.create(
|
||||
api.RepositoryApi.create(
|
||||
self.app_args, parsed_args.type, parsed_args.arch
|
||||
),
|
||||
parsed_args
|
||||
)
|
||||
|
||||
@abc.abstractmethod
|
||||
def take_repo_action(self, api, parsed_args):
|
||||
def take_repo_action(self, repo_api, parsed_args):
|
||||
"""Takes action on repository.
|
||||
|
||||
:param api: the RepositoryApi instance
|
||||
:param repo_api: the RepositoryApi instance
|
||||
:param parsed_args: the command-line arguments
|
||||
:return: the action result
|
||||
"""
|
||||
@ -104,32 +105,13 @@ class PackagesMixin(object):
|
||||
def get_parser(self, prog_name):
|
||||
parser = super(PackagesMixin, self).get_parser(prog_name)
|
||||
parser.add_argument(
|
||||
"--skip-mandatory",
|
||||
dest='include_mandatory',
|
||||
action='store_false',
|
||||
default=True,
|
||||
help="Do not copy mandatory packages."
|
||||
)
|
||||
|
||||
group = parser.add_mutually_exclusive_group()
|
||||
|
||||
group.add_argument(
|
||||
"-p", "--packages",
|
||||
"-R", "--requirements",
|
||||
dest='requirements',
|
||||
type=read_from_file,
|
||||
metavar='FILENAME',
|
||||
help="The path to file with list of packages."
|
||||
"See documentation about format."
|
||||
)
|
||||
|
||||
group.add_argument(
|
||||
"-f", "--exclude-filter",
|
||||
dest='exclude_filter_data',
|
||||
type=read_from_file,
|
||||
metavar='FILENAME',
|
||||
help="The path to file with package exclude filter data."
|
||||
"See documentation about format."
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
|
@ -47,15 +47,15 @@ class CloneCommand(PackagesMixin, RepositoriesMixin, BaseRepoCommand):
|
||||
|
||||
return parser
|
||||
|
||||
def take_repo_action(self, api, parsed_args):
|
||||
stat = api.clone_repositories(
|
||||
def take_repo_action(self, repo_api, parsed_args):
|
||||
stat = repo_api.clone_repositories(
|
||||
parsed_args.repositories,
|
||||
parsed_args.requirements,
|
||||
parsed_args.destination,
|
||||
parsed_args.sources,
|
||||
parsed_args.locales,
|
||||
parsed_args.include_mandatory,
|
||||
filter_data=parsed_args.exclude_filter_data,
|
||||
parsed_args.requirements,
|
||||
repo_api.CopyOptions(
|
||||
sources=parsed_args.sources,
|
||||
localizations=parsed_args.locales,
|
||||
)
|
||||
)
|
||||
self.stdout.write(
|
||||
"Packages copied: {0.copied}/{0.total}.\n".format(stat)
|
||||
|
@ -43,8 +43,8 @@ class CreateCommand(BaseRepoCommand):
|
||||
)
|
||||
return parser
|
||||
|
||||
def take_repo_action(self, api, parsed_args):
|
||||
api.create_repository(
|
||||
def take_repo_action(self, repo_api, parsed_args):
|
||||
repo_api.create_repository(
|
||||
parsed_args.repository,
|
||||
parsed_args.package_files
|
||||
)
|
||||
|
@ -22,7 +22,8 @@ from packetary.cli.commands.base import RepositoriesMixin
|
||||
|
||||
|
||||
class ListOfPackages(
|
||||
PackagesMixin, RepositoriesMixin, BaseProduceOutputCommand):
|
||||
PackagesMixin, RepositoriesMixin, BaseProduceOutputCommand):
|
||||
|
||||
"""Gets the list of packages from repository(es)."""
|
||||
|
||||
columns = (
|
||||
@ -37,12 +38,10 @@ class ListOfPackages(
|
||||
"requires",
|
||||
)
|
||||
|
||||
def take_repo_action(self, api, parsed_args):
|
||||
return api.get_packages(
|
||||
def take_repo_action(self, repo_api, parsed_args):
|
||||
return repo_api.get_packages(
|
||||
parsed_args.repositories,
|
||||
parsed_args.requirements,
|
||||
parsed_args.include_mandatory,
|
||||
filter_data=parsed_args.exclude_filter_data,
|
||||
parsed_args.requirements
|
||||
)
|
||||
|
||||
|
||||
|
@ -29,8 +29,8 @@ class ListOfUnresolved(RepositoriesMixin, BaseProduceOutputCommand):
|
||||
"alternative",
|
||||
)
|
||||
|
||||
def take_repo_action(self, api, parsed_args):
|
||||
return api.get_unresolved_dependencies(
|
||||
def take_repo_action(self, repo_api, parsed_args):
|
||||
return repo_api.get_unresolved_dependencies(
|
||||
parsed_args.repositories
|
||||
)
|
||||
|
||||
|
@ -84,13 +84,12 @@ class RepositoryController(object):
|
||||
connection = self.context.connection
|
||||
self.driver.get_packages(connection, repository, consumer)
|
||||
|
||||
def fork_repository(self, repository, destination, source, locale):
|
||||
def fork_repository(self, repository, destination, options):
|
||||
"""Creates copy of repositories.
|
||||
|
||||
:param repository: the origin repository
|
||||
:param destination: the target folder
|
||||
:param source: If True, the source packages will be copied too.
|
||||
:param locale: If True, the localisation will be copied too.
|
||||
:param options: The options, see RepositoryCopyOptions
|
||||
:return: the mapping origin to cloned repository.
|
||||
"""
|
||||
new_path = os.path.join(
|
||||
@ -101,7 +100,7 @@ class RepositoryController(object):
|
||||
)
|
||||
logger.info("cloning repository '%s' to '%s'", repository, new_path)
|
||||
return self.driver.fork_repository(
|
||||
self.context.connection, repository, new_path, source, locale
|
||||
self.context.connection, repository, new_path, options
|
||||
)
|
||||
|
||||
def assign_packages(self, repository, packages, observer=None):
|
||||
|
@ -55,15 +55,13 @@ class RepositoryDriverBase(object):
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def fork_repository(self, connection, repository, destination,
|
||||
source=False, locale=False):
|
||||
def fork_repository(self, connection, repository, destination, options):
|
||||
"""Creates the new repository with same metadata.
|
||||
|
||||
:param connection: the connection manager instance
|
||||
:param repository: the source repository
|
||||
:param destination: the destination folder
|
||||
:param source: copy source files
|
||||
:param locale: copy localisation
|
||||
:param options: the options
|
||||
:return: The copy of repository
|
||||
"""
|
||||
|
||||
|
@ -197,8 +197,7 @@ class DebRepositoryDriver(RepositoryDriverBase):
|
||||
self.logger.info("saved %d packages in %s", count, repository)
|
||||
self._update_suite_index(repository)
|
||||
|
||||
def fork_repository(self, connection, repository, destination,
|
||||
source=False, locale=False):
|
||||
def fork_repository(self, connection, repository, destination, options):
|
||||
# TODO(download gpk)
|
||||
# TODO(sources and locales)
|
||||
new_repo = copy.copy(repository)
|
||||
|
@ -162,8 +162,7 @@ class RpmRepositoryDriver(RepositoryDriverBase):
|
||||
groupstree = self._load_groups(connection, repository)
|
||||
self._rebuild_repository(connection, repository, packages, groupstree)
|
||||
|
||||
def fork_repository(self, connection, repository, destination,
|
||||
source=False, locale=False):
|
||||
def fork_repository(self, connection, repository, destination, options):
|
||||
# TODO(download gpk)
|
||||
# TODO(sources and locales)
|
||||
new_repo = copy.copy(repository)
|
||||
|
26
packetary/library/functions.py
Normal file
26
packetary/library/functions.py
Normal file
@ -0,0 +1,26 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
|
||||
def compose(*functions):
|
||||
"""Call all functions with same arguments."""
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
for f in functions:
|
||||
f(*args, **kwargs)
|
||||
return wrapper
|
@ -18,14 +18,12 @@
|
||||
|
||||
from packetary.schemas.deb_repo_schema import DEB_REPO_SCHEMA
|
||||
from packetary.schemas.package_files_schema import PACKAGE_FILES_SCHEMA
|
||||
from packetary.schemas.package_filter_schema import PACKAGE_FILTER_SCHEMA
|
||||
from packetary.schemas.packages_schema import PACKAGES_SCHEMA
|
||||
from packetary.schemas.requirements_schema import REQUIREMENTS_SCHEMA
|
||||
from packetary.schemas.rpm_repo_schema import RPM_REPO_SCHEMA
|
||||
|
||||
__all__ = [
|
||||
"DEB_REPO_SCHEMA",
|
||||
"PACKAGE_FILES_SCHEMA",
|
||||
"PACKAGE_FILTER_SCHEMA",
|
||||
"PACKAGES_SCHEMA",
|
||||
"REQUIREMENTS_SCHEMA",
|
||||
"RPM_REPO_SCHEMA",
|
||||
]
|
||||
|
74
packetary/schemas/requirements_schema.py
Normal file
74
packetary/schemas/requirements_schema.py
Normal file
@ -0,0 +1,74 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
REQUIREMENTS_SCHEMA = {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"type": "object",
|
||||
"anyOf": [
|
||||
{"required": ["packages"]},
|
||||
{"required": ["repositories"]},
|
||||
{"required": ["mandatory"]}
|
||||
],
|
||||
"properties": {
|
||||
"repositories": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["name"],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"excludes": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"patternProperties": {
|
||||
r"[a-z][\w_]*": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"packages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["name"],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"versions": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"pattern": "^([<>]=?|=)\s+.+$"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"mandatory": {
|
||||
"type": "boolean"
|
||||
},
|
||||
}
|
||||
}
|
@ -26,6 +26,8 @@ class TestCase(unittest.TestCase):
|
||||
|
||||
"""Test case base class for all unit tests."""
|
||||
|
||||
maxDiff = None
|
||||
|
||||
def _check_cases(self, assertion, cases, method):
|
||||
for exp, value in cases:
|
||||
assertion(
|
||||
|
@ -36,11 +36,9 @@ class CallbacksAdapter(mock.MagicMock):
|
||||
else:
|
||||
callback = None
|
||||
|
||||
if not callable(callback):
|
||||
return super(CallbacksAdapter, self).__call__(*args, **kwargs)
|
||||
|
||||
args = args[:-1]
|
||||
data = super(CallbacksAdapter, self).__call__(*args, **kwargs)
|
||||
if not callable(callback):
|
||||
return data
|
||||
|
||||
if isinstance(data, list):
|
||||
for d in data:
|
||||
|
57
packetary/tests/test_api_context.py
Normal file
57
packetary/tests/test_api_context.py
Normal file
@ -0,0 +1,57 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
import mock
|
||||
|
||||
from packetary.api import context
|
||||
|
||||
from packetary.tests import base
|
||||
|
||||
|
||||
class TestContext(base.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.config = context.Configuration(
|
||||
threads_num=2,
|
||||
ignore_errors_num=3,
|
||||
retries_num=5,
|
||||
retry_interval=10,
|
||||
http_proxy="http://localhost",
|
||||
https_proxy="https://localhost"
|
||||
)
|
||||
|
||||
@mock.patch("packetary.api.context.ConnectionsManager")
|
||||
def test_initialise_connection_manager(self, conn_manager):
|
||||
ctx = context.Context(self.config)
|
||||
conn_manager.assert_called_once_with(
|
||||
proxy="http://localhost",
|
||||
secure_proxy="https://localhost",
|
||||
retries_num=5,
|
||||
retry_interval=10
|
||||
)
|
||||
|
||||
self.assertIs(conn_manager(), ctx.connection)
|
||||
|
||||
@mock.patch("packetary.api.context.AsynchronousSection")
|
||||
def test_asynchronous_section(self, async_section):
|
||||
ctx = context.Context(self.config)
|
||||
s = ctx.async_section()
|
||||
async_section.assert_called_with(2, 3)
|
||||
self.assertIs(s, async_section())
|
||||
ctx.async_section(0)
|
||||
async_section.assert_called_with(2, 0)
|
84
packetary/tests/test_api_loaders.py
Normal file
84
packetary/tests/test_api_loaders.py
Normal file
@ -0,0 +1,84 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
from packetary.api import loaders
|
||||
|
||||
from packetary.tests import base
|
||||
from packetary.tests.stubs import generator
|
||||
|
||||
|
||||
class TestDataLoaders(base.TestCase):
|
||||
def test_load_filter(self):
|
||||
filter_data = [
|
||||
{"name": "p1", "group": "g1"},
|
||||
{"name": "p2"},
|
||||
{"group": "g3"},
|
||||
{"name": "/^.5/", "group": "/^.*3/"},
|
||||
{"group": "/^.*4/"},
|
||||
]
|
||||
filters = loaders.load_filters(filter_data)
|
||||
cases = [
|
||||
(True, (generator.gen_package(name='p1', group='g1'),)),
|
||||
(True, (generator.gen_package(name="p2", group="g1"),)),
|
||||
(False, (generator.gen_package(name="p3", group="g2"),)),
|
||||
(True, (generator.gen_package(name="p4", group="g3"),)),
|
||||
(True, (generator.gen_package(name="p5", group="g3"),)),
|
||||
(True, (generator.gen_package(name="p6", group="g4"),)),
|
||||
]
|
||||
self._check_cases(self.assertIs, cases, filters)
|
||||
self.assertFalse(loaders.load_filters([])(cases[0][1][0]))
|
||||
|
||||
def test_load_package_relations(self):
|
||||
data = [
|
||||
{'name': 'test1'},
|
||||
{'name': 'test2', 'versions': ['> 1', '< 3']},
|
||||
]
|
||||
expected = [
|
||||
str(generator.gen_relation('test1')),
|
||||
str(generator.gen_relation('test2', ['<', '3'])),
|
||||
str(generator.gen_relation('test2', ['>', '1'])),
|
||||
]
|
||||
actual = []
|
||||
loaders.load_package_relations(data, lambda x: actual.append(str(x)))
|
||||
self.assertItemsEqual(expected, actual)
|
||||
actual = []
|
||||
loaders.load_package_relations(None, actual.append)
|
||||
self.assertEqual([], actual)
|
||||
|
||||
def test_get_packages_traverse(self):
|
||||
data = [{
|
||||
'name': 'r1',
|
||||
'excludes': [{'name': 'p1'}]
|
||||
}]
|
||||
repo = generator.gen_repository(name='r1')
|
||||
repo2 = generator.gen_repository(name='r2')
|
||||
packages = [
|
||||
generator.gen_package(name='p1', version=1, repository=repo),
|
||||
generator.gen_package(name='p2', version=2, repository=repo),
|
||||
generator.gen_package(name='p3', version=2, repository=repo2),
|
||||
generator.gen_package(name='p4', version=2, repository=repo2)
|
||||
]
|
||||
actual = []
|
||||
traverse = loaders.get_packages_traverse(
|
||||
data, lambda x: actual.append(str(x))
|
||||
)
|
||||
for p in packages:
|
||||
traverse(p)
|
||||
|
||||
expected = [str(generator.gen_relation('p2', ['=', '2']))]
|
||||
self.assertItemsEqual(expected, actual)
|
91
packetary/tests/test_api_validators.py
Normal file
91
packetary/tests/test_api_validators.py
Normal file
@ -0,0 +1,91 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
import mock
|
||||
|
||||
from jsonschema import SchemaError
|
||||
from jsonschema import ValidationError
|
||||
|
||||
from packetary.api import validators
|
||||
|
||||
from packetary.tests import base
|
||||
|
||||
|
||||
@mock.patch('packetary.api.validators.jsonschema',
|
||||
ValidationError=ValidationError, SchemaError=SchemaError)
|
||||
class TestDataValidators(base.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.data = {'key': 'value'}
|
||||
cls.schema = {'type': 'object'}
|
||||
|
||||
def test_validate_data(self, jsonschema_mock):
|
||||
validators._validate_data(self.data, self.schema)
|
||||
jsonschema_mock.validate.assert_called_once_with(
|
||||
self.data, self.schema
|
||||
)
|
||||
|
||||
def test_validate_invalid_data(self, jsonschema_mock):
|
||||
paths = [(("a", 0), "\['a'\]\[0\]"), ((), "")]
|
||||
for path, details in paths:
|
||||
msg = "Invalid data: error."
|
||||
if details:
|
||||
msg += "\nField: {0}".format(details)
|
||||
with self.assertRaisesRegexp(ValueError, msg):
|
||||
jsonschema_mock.validate.side_effect = ValidationError(
|
||||
"error", path=path
|
||||
)
|
||||
validators._validate_data(self.data, self.schema)
|
||||
|
||||
msg = "Invalid schema: error."
|
||||
if details:
|
||||
msg += "\nField: {0}".format(details)
|
||||
with self.assertRaisesRegexp(ValueError, msg):
|
||||
jsonschema_mock.validate.side_effect = SchemaError(
|
||||
"error", schema_path=path
|
||||
)
|
||||
validators._validate_data(self.data, self.schema)
|
||||
|
||||
def test_build_validator(self, jsonschema_mock):
|
||||
schemas = [
|
||||
lambda this: this.schema,
|
||||
lambda: self.schema,
|
||||
self.schema
|
||||
]
|
||||
for schema in schemas:
|
||||
jsonschema_mock.reset()
|
||||
validator = validators._build_validator(schema)
|
||||
validator(self, self.data)
|
||||
jsonschema_mock.validate.assert_called_with(self.data, self.schema)
|
||||
|
||||
def test_declare_schema_default_does_not_check(self, jsonschema_mock):
|
||||
func = validators.declare_schema(p=self.schema)(lambda p=None: None)
|
||||
func(None)
|
||||
self.assertEqual(0, jsonschema_mock.validate.call_count)
|
||||
|
||||
def test_declare_schema_check_data(self, jsonschema_mock):
|
||||
func = validators.declare_schema(p=self.schema)(lambda p: None)
|
||||
func(self.data)
|
||||
jsonschema_mock.validate.assert_called_with(self.data, self.schema)
|
||||
|
||||
def test_declare_schema_if_schema_is_method(self, jsonschema_mock):
|
||||
func = validators.declare_schema(p=lambda x: x.schema)(
|
||||
lambda x, p: None
|
||||
)
|
||||
func(self, self.data)
|
||||
jsonschema_mock.validate.assert_called_with(self.data, self.schema)
|
@ -16,20 +16,22 @@
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
import mock
|
||||
import subprocess
|
||||
|
||||
import mock
|
||||
|
||||
# The cmd2 does not work with python3.5
|
||||
# because it tries to get access to the property mswindows,
|
||||
# that was removed in 3.5
|
||||
subprocess.mswindows = False
|
||||
|
||||
from packetary.api import RepositoryApi
|
||||
from packetary.api.statistics import CopyStatistics
|
||||
from packetary.cli.commands import clone
|
||||
from packetary.cli.commands import create
|
||||
from packetary.cli.commands import packages
|
||||
from packetary.cli.commands import unresolved
|
||||
from packetary.objects.statistics import CopyStatistics
|
||||
|
||||
from packetary.tests import base
|
||||
from packetary.tests.stubs.generator import gen_package
|
||||
from packetary.tests.stubs.generator import gen_relation
|
||||
@ -38,7 +40,7 @@ from packetary.tests.stubs.generator import gen_repository
|
||||
|
||||
@mock.patch("packetary.cli.commands.base.BaseRepoCommand.stdout")
|
||||
@mock.patch("packetary.cli.commands.base.read_from_file")
|
||||
@mock.patch("packetary.cli.commands.base.RepositoryApi")
|
||||
@mock.patch("packetary.cli.commands.base.api.RepositoryApi")
|
||||
class TestCliCommands(base.TestCase):
|
||||
common_argv = [
|
||||
"--ignore-errors-num=3",
|
||||
@ -51,11 +53,10 @@ class TestCliCommands(base.TestCase):
|
||||
|
||||
clone_argv = [
|
||||
"-r", "repositories.yaml",
|
||||
"-p", "packages.yaml",
|
||||
"-R", "requirements.yaml",
|
||||
"-d", "/root",
|
||||
"-t", "deb",
|
||||
"-a", "x86_64",
|
||||
"--skip-mandatory"
|
||||
]
|
||||
|
||||
create_argv = [
|
||||
@ -95,7 +96,7 @@ class TestCliCommands(base.TestCase):
|
||||
def test_clone_cmd(self, api_mock, read_file_mock, stdout_mock):
|
||||
read_file_mock.side_effect = [
|
||||
[{"name": "repo"}],
|
||||
[{"name": "package"}],
|
||||
{'packages': [{"name": "package"}]},
|
||||
]
|
||||
api_instance = self.get_api_instance_mock(api_mock)
|
||||
api_instance.clone_repositories.return_value = CopyStatistics()
|
||||
@ -105,13 +106,13 @@ class TestCliCommands(base.TestCase):
|
||||
)
|
||||
self.check_common_config(api_mock.create.call_args[0][0])
|
||||
read_file_mock.assert_any_call("repositories.yaml")
|
||||
read_file_mock.assert_any_call("packages.yaml")
|
||||
read_file_mock.assert_any_call("requirements.yaml")
|
||||
api_instance.clone_repositories.assert_called_once_with(
|
||||
[{"name": "repo"}], [{"name": "package"}], "/root",
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
filter_data=None,
|
||||
[{"name": "repo"}], "/root", {'packages': [{"name": "package"}]},
|
||||
api_instance.CopyOptions.return_value
|
||||
)
|
||||
api_instance.CopyOptions.assert_called_once_with(
|
||||
sources=False, localizations=False,
|
||||
)
|
||||
stdout_mock.write.assert_called_once_with(
|
||||
"Packages copied: 0/0.\n"
|
||||
@ -132,7 +133,7 @@ class TestCliCommands(base.TestCase):
|
||||
)
|
||||
self.check_common_config(api_mock.create.call_args[0][0])
|
||||
api_instance.get_packages.assert_called_once_with(
|
||||
[{"name": "repo"}], None, True, filter_data=None
|
||||
[{"name": "repo"}], None
|
||||
)
|
||||
self.assertIn(
|
||||
"test1; test1.pkg",
|
||||
|
@ -230,7 +230,7 @@ class TestDebDriver(base.TestCase):
|
||||
]
|
||||
open.side_effect = files
|
||||
new_repo = self.driver.fork_repository(
|
||||
self.connection, self.repo, "/root/test"
|
||||
self.connection, self.repo, "/root/test", None
|
||||
)
|
||||
self.assertEqual(self.repo.name, new_repo.name)
|
||||
self.assertEqual(self.repo.architecture, new_repo.architecture)
|
||||
|
@ -19,62 +19,77 @@
|
||||
import copy
|
||||
import mock
|
||||
|
||||
import jsonschema
|
||||
from packetary import api
|
||||
from packetary.controllers.repository import RepositoryController
|
||||
from packetary import schemas
|
||||
|
||||
from packetary.api import Configuration
|
||||
from packetary.api import Context
|
||||
from packetary.api import RepositoryApi
|
||||
from packetary.schemas import PACKAGE_FILES_SCHEMA
|
||||
from packetary.schemas import PACKAGE_FILTER_SCHEMA
|
||||
from packetary.schemas import PACKAGES_SCHEMA
|
||||
from packetary.tests import base
|
||||
from packetary.tests.stubs import generator
|
||||
from packetary.tests.stubs.helpers import CallbacksAdapter
|
||||
|
||||
|
||||
@mock.patch("packetary.api.jsonschema")
|
||||
@mock.patch("packetary.api.validators.jsonschema")
|
||||
class TestRepositoryApi(base.TestCase):
|
||||
def setUp(self):
|
||||
self.controller = CallbacksAdapter()
|
||||
self.api = RepositoryApi(self.controller)
|
||||
self.repo_data = {"name": "repo1", "uri": "file:///repo1"}
|
||||
self.requirements_data = [
|
||||
{"name": "test1"}, {"name": "test2", "versions": ["< 3", "> 1"]}
|
||||
]
|
||||
self.controller = CallbacksAdapter(spec=RepositoryController)
|
||||
self.api = api.RepositoryApi(self.controller)
|
||||
self.schema = {}
|
||||
self.repo = generator.gen_repository(**self.repo_data)
|
||||
self.controller.load_repositories.return_value = [self.repo]
|
||||
self.controller.get_repository_data_schema.return_value = self.schema
|
||||
self._generate_packages()
|
||||
|
||||
def _generate_repositories(self, count=1):
|
||||
self.repos_data = [
|
||||
{"name": "repo{0}".format(i), "uri": "file:///repo{0}".format(i)}
|
||||
for i in range(count)
|
||||
]
|
||||
self.repos = [
|
||||
generator.gen_repository(**data) for data in self.repos_data
|
||||
]
|
||||
self.controller.load_repositories.return_value = self.repos
|
||||
|
||||
def _generate_mirrors(self):
|
||||
mirrors = {}
|
||||
for repo in self.repos:
|
||||
mirror = copy.copy(repo)
|
||||
mirror.url = "file:///mirror/{0}".format(repo.name)
|
||||
mirrors[repo] = mirror
|
||||
self.controller.fork_repository.side_effect = lambda *x: mirrors[x[0]]
|
||||
self.mirrors = mirrors
|
||||
|
||||
def _generate_packages(self):
|
||||
self.packages = [
|
||||
generator.gen_package(idx=1, repository=self.repo, requires=None),
|
||||
generator.gen_package(idx=2, repository=self.repo, requires=None),
|
||||
generator.gen_package(
|
||||
idx=3, repository=self.repo, mandatory=True,
|
||||
requires=[generator.gen_relation("package2")]
|
||||
),
|
||||
generator.gen_package(
|
||||
idx=4, repository=self.repo, mandatory=False,
|
||||
requires=[generator.gen_relation("package1")]
|
||||
),
|
||||
generator.gen_package(
|
||||
idx=5, repository=self.repo,
|
||||
requires=[generator.gen_relation("package6")])
|
||||
[
|
||||
generator.gen_package(
|
||||
name='{0}_1'.format(r.name), repository=r, requires=None
|
||||
),
|
||||
generator.gen_package(
|
||||
name='{0}_2'.format(r.name), repository=r, requires=None
|
||||
),
|
||||
generator.gen_package(
|
||||
name='{0}_3'.format(r.name), repository=r, mandatory=True,
|
||||
requires=[generator.gen_relation("{0}_2".format(r.name))]
|
||||
),
|
||||
generator.gen_package(
|
||||
name='{0}_4'.format(r.name), repository=r, mandatory=False,
|
||||
requires=[generator.gen_relation("{0}_1".format(r.name))]
|
||||
),
|
||||
generator.gen_package(
|
||||
name='{0}_5'.format(r.name), repository=r,
|
||||
requires=[generator.gen_relation("unresolved")]
|
||||
),
|
||||
]
|
||||
for r in self.repos
|
||||
]
|
||||
self.controller.load_packages.return_value = self.packages
|
||||
self.controller.load_packages.side_effect = self.packages
|
||||
|
||||
@mock.patch("packetary.api.RepositoryController")
|
||||
@mock.patch("packetary.api.ConnectionsManager")
|
||||
def test_create_with_config(self, connection_mock, controller_mock,
|
||||
jsonschema_mock):
|
||||
config = Configuration(
|
||||
@mock.patch("packetary.api.context.ConnectionsManager")
|
||||
@mock.patch("packetary.api.repositories.RepositoryController")
|
||||
def test_create_with_config(self, controller_mock, connection_mock, _):
|
||||
config = api.Configuration(
|
||||
http_proxy="http://localhost", https_proxy="https://localhost",
|
||||
retries_num=10, retry_interval=1, threads_num=8,
|
||||
ignore_errors_num=6
|
||||
)
|
||||
RepositoryApi.create(config, "deb", "x86_64")
|
||||
api.RepositoryApi.create(config, "deb", "x86_64")
|
||||
connection_mock.assert_called_once_with(
|
||||
proxy="http://localhost",
|
||||
secure_proxy="https://localhost",
|
||||
@ -85,17 +100,16 @@ class TestRepositoryApi(base.TestCase):
|
||||
mock.ANY, "deb", "x86_64"
|
||||
)
|
||||
|
||||
@mock.patch("packetary.api.RepositoryController")
|
||||
@mock.patch("packetary.api.ConnectionsManager")
|
||||
def test_create_with_context(self, connection_mock, controller_mock,
|
||||
jsonschema_mock):
|
||||
config = Configuration(
|
||||
@mock.patch("packetary.api.context.ConnectionsManager")
|
||||
@mock.patch("packetary.api.repositories.RepositoryController")
|
||||
def test_create_with_context(self, controller_mock, connection_mock, _):
|
||||
config = api.Configuration(
|
||||
http_proxy="http://localhost", https_proxy="https://localhost",
|
||||
retries_num=10, retry_interval=1, threads_num=8,
|
||||
ignore_errors_num=6
|
||||
)
|
||||
context = Context(config)
|
||||
RepositoryApi.create(context, "deb", "x86_64")
|
||||
context = api.Context(config)
|
||||
api.RepositoryApi.create(context, "deb", "x86_64")
|
||||
connection_mock.assert_called_once_with(
|
||||
proxy="http://localhost",
|
||||
secure_proxy="https://localhost",
|
||||
@ -108,299 +122,109 @@ class TestRepositoryApi(base.TestCase):
|
||||
|
||||
def test_create_repository(self, jsonschema_mock):
|
||||
file_urls = ["file://test1.pkg"]
|
||||
self.api.create_repository(self.repo_data, file_urls)
|
||||
self._generate_repositories(1)
|
||||
self.api.create_repository(self.repos_data[0], file_urls)
|
||||
self.controller.create_repository.assert_called_once_with(
|
||||
self.repo_data, file_urls
|
||||
)
|
||||
jsonschema_mock.validate.assert_has_calls(
|
||||
[
|
||||
mock.call(self.repo_data, self.schema),
|
||||
mock.call(file_urls, PACKAGE_FILES_SCHEMA),
|
||||
]
|
||||
self.repos_data[0], file_urls
|
||||
)
|
||||
jsonschema_mock.validate.assert_has_calls([
|
||||
mock.call(self.repos_data[0], self.schema),
|
||||
mock.call(file_urls, schemas.PACKAGE_FILES_SCHEMA),
|
||||
], any_order=True)
|
||||
|
||||
def test_get_packages_as_is(self, jsonschema_mock):
|
||||
packages = self.api.get_packages([self.repo_data], None, False, None)
|
||||
self.assertEqual(5, len(packages))
|
||||
self.assertItemsEqual(
|
||||
self.packages,
|
||||
packages
|
||||
)
|
||||
self._generate_repositories(1)
|
||||
self._generate_packages()
|
||||
packages = self.api.get_packages(self.repos_data)
|
||||
self.assertEqual(5, len(self.packages[0]))
|
||||
self.assertItemsEqual(self.packages[0], packages)
|
||||
jsonschema_mock.validate.assert_called_once_with(
|
||||
self.repo_data, self.schema
|
||||
self.repos_data, self.api._get_repositories_data_schema()
|
||||
)
|
||||
|
||||
def test_get_packages_by_requirements_with_mandatory(self,
|
||||
jsonschema_mock):
|
||||
requirements = [{"name": "package1"}]
|
||||
packages = self.api.get_packages(
|
||||
[self.repo_data], requirements, True, None
|
||||
)
|
||||
self.assertEqual(3, len(packages))
|
||||
def test_get_packages_by_requirements(self, jsonschema_mock):
|
||||
self._generate_repositories(2)
|
||||
self._generate_packages()
|
||||
requirements = {
|
||||
'packages': [{"name": "repo0_1"}],
|
||||
'repositories': [{"name": "repo1"}],
|
||||
'mandatory': True
|
||||
}
|
||||
packages = self.api.get_packages(self.repos_data, requirements)
|
||||
expected_packages = self.packages[0][:3] + self.packages[1]
|
||||
self.assertItemsEqual(
|
||||
["package1", "package2", "package3"],
|
||||
(x.name for x in packages)
|
||||
)
|
||||
jsonschema_mock.validate.assert_has_calls(
|
||||
[
|
||||
mock.call(self.repo_data, self.schema),
|
||||
mock.call(requirements, PACKAGES_SCHEMA),
|
||||
]
|
||||
)
|
||||
|
||||
def test_get_packages_by_requirements_without_mandatory(self,
|
||||
jsonschema_mock):
|
||||
requirements = [{"name": "package4"}]
|
||||
packages = self.api.get_packages(
|
||||
[self.repo_data], requirements, False, None
|
||||
)
|
||||
self.assertEqual(2, len(packages))
|
||||
self.assertItemsEqual(
|
||||
["package1", "package4"],
|
||||
(x.name for x in packages)
|
||||
)
|
||||
jsonschema_mock.validate.assert_has_calls(
|
||||
[
|
||||
mock.call(self.repo_data, self.schema),
|
||||
mock.call(requirements, PACKAGES_SCHEMA),
|
||||
]
|
||||
[x.name for x in expected_packages],
|
||||
[x.name for x in packages]
|
||||
)
|
||||
repos_schema = self.api._get_repositories_data_schema()
|
||||
jsonschema_mock.validate.assert_has_calls([
|
||||
mock.call(self.repos_data, repos_schema),
|
||||
mock.call(requirements, schemas.REQUIREMENTS_SCHEMA)
|
||||
], any_order=True)
|
||||
|
||||
def test_clone_repositories_as_is(self, jsonschema_mock):
|
||||
# return value is used as statistics
|
||||
mirror = copy.copy(self.repo)
|
||||
mirror.url = "file:///mirror/repo"
|
||||
self.controller.fork_repository.return_value = mirror
|
||||
self._generate_repositories(1)
|
||||
self._generate_packages()
|
||||
self._generate_mirrors()
|
||||
|
||||
self.controller.assign_packages.return_value = [0, 1, 1, 1, 0, 6]
|
||||
stats = self.api.clone_repositories([self.repo_data], None, "/mirror")
|
||||
options = api.RepositoryCopyOptions()
|
||||
stats = self.api.clone_repositories(
|
||||
self.repos_data, "/mirror", options=options)
|
||||
self.controller.fork_repository.assert_called_once_with(
|
||||
self.repo, '/mirror', False, False
|
||||
self.repos[0], '/mirror', options
|
||||
)
|
||||
self.controller.assign_packages.assert_called_once_with(
|
||||
mirror, set(self.packages)
|
||||
self.mirrors[self.repos[0]], set(self.packages[0]), mock.ANY
|
||||
)
|
||||
self.assertEqual(6, stats.total)
|
||||
self.assertEqual(4, stats.copied)
|
||||
jsonschema_mock.validate.assert_called_once_with(
|
||||
self.repo_data, self.schema
|
||||
self.repos_data, self.api._get_repositories_data_schema()
|
||||
)
|
||||
|
||||
def test_clone_by_requirements_with_mandatory(self, jsonschema_mock):
|
||||
# return value is used as statistics
|
||||
mirror = copy.copy(self.repo)
|
||||
mirror.url = "file:///mirror/repo"
|
||||
requirements = [{"name": "package1"}]
|
||||
self.controller.fork_repository.return_value = mirror
|
||||
self.controller.assign_packages.return_value = [0, 1, 1]
|
||||
def test_clone_by_requirements(self, jsonschema_mock):
|
||||
self._generate_repositories(2)
|
||||
self._generate_packages()
|
||||
self._generate_mirrors()
|
||||
requirements = {
|
||||
'packages': [{"name": "repo0_1"}],
|
||||
'repositories': [{"name": "repo1"}],
|
||||
'mandatory': False
|
||||
}
|
||||
self.controller.assign_packages.return_value = [0, 1, 1] * 3
|
||||
stats = self.api.clone_repositories(
|
||||
[self.repo_data], requirements,
|
||||
"/mirror", include_mandatory=True
|
||||
self.repos_data, "/mirror", requirements
|
||||
)
|
||||
packages = {self.packages[0], self.packages[1], self.packages[2]}
|
||||
self.controller.fork_repository.assert_called_once_with(
|
||||
self.repo, '/mirror', False, False
|
||||
self.controller.fork_repository.assert_has_calls(
|
||||
[mock.call(r, '/mirror', mock.ANY) for r in self.repos],
|
||||
any_order=True
|
||||
)
|
||||
self.controller.assign_packages.assert_called_once_with(
|
||||
mirror, packages
|
||||
)
|
||||
self.assertEqual(3, stats.total)
|
||||
self.assertEqual(2, stats.copied)
|
||||
jsonschema_mock.validate.assert_has_calls(
|
||||
[
|
||||
mock.call(self.repo_data, self.schema),
|
||||
mock.call(requirements, PACKAGES_SCHEMA),
|
||||
]
|
||||
)
|
||||
|
||||
def test_clone_by_requirements_without_mandatory(self,
|
||||
jsonschema_mock):
|
||||
# return value is used as statistics
|
||||
mirror = copy.copy(self.repo)
|
||||
mirror.url = "file:///mirror/repo"
|
||||
requirements = [{"name": "package4"}]
|
||||
self.controller.fork_repository.return_value = mirror
|
||||
self.controller.assign_packages.return_value = [0, 4]
|
||||
stats = self.api.clone_repositories(
|
||||
[self.repo_data], requirements,
|
||||
"/mirror", include_mandatory=False
|
||||
)
|
||||
packages = {self.packages[0], self.packages[3]}
|
||||
self.controller.fork_repository.assert_called_once_with(
|
||||
self.repo, '/mirror', False, False
|
||||
)
|
||||
self.controller.assign_packages.assert_called_once_with(
|
||||
mirror, packages
|
||||
)
|
||||
self.assertEqual(2, stats.total)
|
||||
self.assertEqual(1, stats.copied)
|
||||
jsonschema_mock.validate.assert_has_calls(
|
||||
[
|
||||
mock.call(self.repo_data, self.schema),
|
||||
mock.call(requirements, PACKAGES_SCHEMA),
|
||||
]
|
||||
)
|
||||
|
||||
def test_clone_with_filter(self, jsonschema_mock):
|
||||
repos_data = "repos_data"
|
||||
requirements_data = "requirements_data"
|
||||
filter_data = "filter_data"
|
||||
repos = "repos"
|
||||
requirements = "requirements"
|
||||
exclude_filter = "exclude_filter"
|
||||
|
||||
self.api._load_repositories = mock.Mock(return_value=repos)
|
||||
self.api._load_requirements = mock.Mock(return_value=requirements)
|
||||
self.api._load_filter = mock.Mock(return_value=exclude_filter)
|
||||
self.api._get_packages = mock.Mock(return_value=set())
|
||||
self.api.controller = mock.Mock()
|
||||
|
||||
self.api.clone_repositories(repos_data, requirements_data,
|
||||
"destination", filter_data=filter_data)
|
||||
|
||||
self.api._load_repositories.assert_called_once_with(repos_data)
|
||||
self.api._load_requirements.assert_called_once_with(requirements_data)
|
||||
self.api._load_filter.assert_called_once_with(filter_data)
|
||||
self.api._get_packages.assert_called_once_with(
|
||||
repos, requirements, False, exclude_filter)
|
||||
|
||||
def test_get_packages_with_exclude_filter(self, jsonschema_mock):
|
||||
exclude_filter = lambda p: any([p == "p1", p == "p3"])
|
||||
self.api._load_packages = CallbacksAdapter()
|
||||
self.api._load_packages.return_value = ["p1", "p2", "p3", "p4"]
|
||||
packages = self.api._get_packages("repos", None, False, exclude_filter)
|
||||
self.assertSetEqual(packages, set(["p2", "p4"]))
|
||||
|
||||
def test_get_packages_without_exclude_filter(self, jsonschema_mock):
|
||||
self.api._load_packages = CallbacksAdapter()
|
||||
self.api._load_packages.return_value = ["p1", "p2"]
|
||||
packages = self.api._get_packages("repos", None, False, None)
|
||||
self.assertSetEqual(packages, set(["p1", "p2"]))
|
||||
self.controller.assign_packages.assert_has_calls([
|
||||
mock.call(
|
||||
self.mirrors[self.repos[0]],
|
||||
set(self.packages[0][:1]),
|
||||
mock.ANY
|
||||
),
|
||||
mock.call(
|
||||
self.mirrors[self.repos[1]],
|
||||
set(self.packages[1]),
|
||||
mock.ANY
|
||||
)
|
||||
], any_order=True)
|
||||
self.assertEqual(18, stats.total)
|
||||
self.assertEqual(12, stats.copied)
|
||||
repos_schema = self.api._get_repositories_data_schema()
|
||||
jsonschema_mock.validate.assert_has_calls([
|
||||
mock.call(self.repos_data, repos_schema),
|
||||
mock.call(requirements, schemas.REQUIREMENTS_SCHEMA)
|
||||
], any_order=True)
|
||||
|
||||
def test_get_unresolved(self, jsonschema_mock):
|
||||
unresolved = self.api.get_unresolved_dependencies([self.repo_data])
|
||||
self.assertItemsEqual(["package6"], (x.name for x in unresolved))
|
||||
self._generate_repositories(1)
|
||||
self._generate_packages()
|
||||
unresolved = self.api.get_unresolved_dependencies(self.repos_data)
|
||||
self.assertItemsEqual(["unresolved"], (x.name for x in unresolved))
|
||||
jsonschema_mock.validate.assert_called_once_with(
|
||||
self.repo_data, self.schema
|
||||
self.repos_data, self.api._get_repositories_data_schema()
|
||||
)
|
||||
|
||||
def test_load_filter_with_none(self, jsonschema_mock):
|
||||
self.assertIsNone(self.api._load_filter(None))
|
||||
|
||||
def test_load_filter(self, jsonschema_mock):
|
||||
self.api._validate_filter_data = mock.Mock()
|
||||
filter_data = [
|
||||
{"name": "p1", "group": "g1"},
|
||||
{"name": "p2"},
|
||||
{"group": "g3"},
|
||||
{"name": "/^.5/", "group": "/^.*3/"},
|
||||
{"group": "/^.*4/"},
|
||||
]
|
||||
exclude_filter = self.api._load_filter(filter_data)
|
||||
|
||||
p1 = generator.gen_package(name="p1", group="g1")
|
||||
p2 = generator.gen_package(name="p2", group="g1")
|
||||
p3 = generator.gen_package(name="p3", group="g2")
|
||||
p4 = generator.gen_package(name="p4", group="g3")
|
||||
p5 = generator.gen_package(name="p5", group="g3")
|
||||
p6 = generator.gen_package(name="p6", group="g4")
|
||||
|
||||
cases = [
|
||||
(True, (p1,)),
|
||||
(True, (p2,)),
|
||||
(False, (p3,)),
|
||||
(True, (p4,)),
|
||||
(True, (p5,)),
|
||||
(True, (p6,)),
|
||||
]
|
||||
self._check_cases(self.assertEqual, cases, exclude_filter)
|
||||
|
||||
def test_validate_filter_data(self, jsonschema_mock):
|
||||
self.api._validate_data = mock.Mock()
|
||||
self.api._validate_filter_data("filter_data")
|
||||
self.api._validate_data.assert_called_once_with("filter_data",
|
||||
PACKAGE_FILTER_SCHEMA)
|
||||
|
||||
def test_load_requirements(self, jsonschema_mock):
|
||||
expected = {
|
||||
generator.gen_relation("test1"),
|
||||
generator.gen_relation("test2", ["<", "3"]),
|
||||
generator.gen_relation("test2", [">", "1"]),
|
||||
}
|
||||
actual = set(self.api._load_requirements(
|
||||
self.requirements_data
|
||||
))
|
||||
self.assertEqual(expected, actual)
|
||||
self.assertIsNone(self.api._load_requirements(None))
|
||||
jsonschema_mock.validate.assert_called_once_with(
|
||||
self.requirements_data,
|
||||
PACKAGES_SCHEMA
|
||||
)
|
||||
|
||||
def test_validate_data(self, jsonschema_mock):
|
||||
self.api._validate_data(self.repo_data, self.schema)
|
||||
jsonschema_mock.validate.assert_called_once_with(
|
||||
self.repo_data, self.schema
|
||||
)
|
||||
|
||||
def test_validate_invalid_data(self, jschema_m):
|
||||
jschema_m.ValidationError = jsonschema.ValidationError
|
||||
jschema_m.SchemaError = jsonschema.SchemaError
|
||||
paths = [(("a", 0), "\['a'\]\[0\]"), ((), "")]
|
||||
for path, details in paths:
|
||||
msg = "Invalid data: error."
|
||||
if details:
|
||||
msg += "\nField: {0}".format(details)
|
||||
with self.assertRaisesRegexp(ValueError, msg):
|
||||
jschema_m.validate.side_effect = jsonschema.ValidationError(
|
||||
"error", path=path
|
||||
)
|
||||
self.api._validate_data([], {})
|
||||
jschema_m.validate.assert_called_with([], {})
|
||||
jschema_m.validate.reset_mock()
|
||||
|
||||
msg = "Invalid schema: error."
|
||||
if details:
|
||||
msg += "\nField: {0}".format(details)
|
||||
with self.assertRaisesRegexp(ValueError, msg):
|
||||
jschema_m.validate.side_effect = jsonschema.SchemaError(
|
||||
"error", schema_path=path
|
||||
)
|
||||
self.api._validate_data([], {})
|
||||
jschema_m.validate.assert_called_with([], {})
|
||||
|
||||
|
||||
class TestContext(base.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.config = Configuration(
|
||||
threads_num=2,
|
||||
ignore_errors_num=3,
|
||||
retries_num=5,
|
||||
retry_interval=10,
|
||||
http_proxy="http://localhost",
|
||||
https_proxy="https://localhost"
|
||||
)
|
||||
|
||||
@mock.patch("packetary.api.ConnectionsManager")
|
||||
def test_initialise_connection_manager(self, conn_manager):
|
||||
context = Context(self.config)
|
||||
conn_manager.assert_called_once_with(
|
||||
proxy="http://localhost",
|
||||
secure_proxy="https://localhost",
|
||||
retries_num=5,
|
||||
retry_interval=10
|
||||
)
|
||||
|
||||
self.assertIs(
|
||||
conn_manager(),
|
||||
context.connection
|
||||
)
|
||||
|
||||
@mock.patch("packetary.api.AsynchronousSection")
|
||||
def test_asynchronous_section(self, async_section):
|
||||
context = Context(self.config)
|
||||
s = context.async_section()
|
||||
async_section.assert_called_with(2, 3)
|
||||
self.assertIs(s, async_section())
|
||||
context.async_section(0)
|
||||
async_section.assert_called_with(2, 0)
|
||||
|
@ -60,7 +60,7 @@ class TestRepositoryController(base.TestCase):
|
||||
|
||||
repos = self.ctrl.load_repositories([repo_data])
|
||||
self.driver.get_repository.assert_called_once_with(
|
||||
self.context.connection, repo_data, self.ctrl.arch
|
||||
self.context.connection, repo_data, self.ctrl.arch, mock.ANY
|
||||
)
|
||||
self.assertEqual([repo], repos)
|
||||
|
||||
@ -93,14 +93,14 @@ class TestRepositoryController(base.TestCase):
|
||||
clone.url = "/root/repo"
|
||||
self.driver.fork_repository.return_value = clone
|
||||
self.context.connection.retrieve.side_effect = [0, 10]
|
||||
self.ctrl.fork_repository(repo, "./repo", False, False)
|
||||
self.ctrl.fork_repository(repo, "./repo", None)
|
||||
self.driver.fork_repository.assert_called_once_with(
|
||||
self.context.connection, repo, "./repo/test", False, False
|
||||
self.context.connection, repo, "./repo/test", None
|
||||
)
|
||||
repo.path = "os"
|
||||
self.ctrl.fork_repository(repo, "./repo", False, False)
|
||||
self.ctrl.fork_repository(repo, "./repo", None)
|
||||
self.driver.fork_repository.assert_called_with(
|
||||
self.context.connection, repo, "./repo/os", False, False
|
||||
self.context.connection, repo, "./repo/os", None
|
||||
)
|
||||
|
||||
def test_copy_packages(self):
|
||||
|
@ -265,7 +265,8 @@ class TestRpmDriver(base.TestCase):
|
||||
new_repo = self.driver.fork_repository(
|
||||
self.connection,
|
||||
repo,
|
||||
"/repo/os/x86_64"
|
||||
"/repo/os/x86_64",
|
||||
None
|
||||
)
|
||||
m_ensure_dir_exists.assert_called_once_with("/repo/os/x86_64")
|
||||
self.assertEqual(repo.name, new_repo.name)
|
||||
@ -310,7 +311,8 @@ class TestRpmDriver(base.TestCase):
|
||||
self.driver.fork_repository(
|
||||
self.connection,
|
||||
repo,
|
||||
"/repo/os/x86_64"
|
||||
"/repo/os/x86_64",
|
||||
None
|
||||
)
|
||||
self.assertEqual(tmp_file.name, md_config.groupfile)
|
||||
os_mock.unlink.assert_called_once_with(tmp_file.name)
|
||||
|
@ -18,14 +18,14 @@
|
||||
|
||||
import jsonschema
|
||||
|
||||
from packetary.schemas import DEB_REPO_SCHEMA
|
||||
from packetary.schemas import PACKAGE_FILES_SCHEMA
|
||||
from packetary.schemas import PACKAGES_SCHEMA
|
||||
from packetary.schemas import RPM_REPO_SCHEMA
|
||||
from packetary import schemas
|
||||
|
||||
from packetary.tests import base
|
||||
|
||||
|
||||
class TestRepositorySchemaBase(base.TestCase):
|
||||
schema = None
|
||||
|
||||
def check_invalid_name(self):
|
||||
self._check_invalid_type('name')
|
||||
|
||||
@ -78,7 +78,7 @@ class TestRepositorySchemaBase(base.TestCase):
|
||||
|
||||
class TestDebRepoSchema(TestRepositorySchemaBase):
|
||||
def setUp(self):
|
||||
self.schema = DEB_REPO_SCHEMA
|
||||
self.schema = schemas.DEB_REPO_SCHEMA
|
||||
|
||||
def test_valid_repo_data(self):
|
||||
repo_data = {
|
||||
@ -142,7 +142,7 @@ class TestDebRepoSchema(TestRepositorySchemaBase):
|
||||
|
||||
class TestRpmRepoSchema(TestRepositorySchemaBase):
|
||||
def setUp(self):
|
||||
self.schema = RPM_REPO_SCHEMA
|
||||
self.schema = schemas.RPM_REPO_SCHEMA
|
||||
|
||||
def test_valid_repo_data(self):
|
||||
repo_data = {
|
||||
@ -170,63 +170,45 @@ class TestRpmRepoSchema(TestRepositorySchemaBase):
|
||||
self.check_invalid_path()
|
||||
|
||||
|
||||
class TestPackagesSchema(base.TestCase):
|
||||
class TestRequirementsSchema(base.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.schema = PACKAGES_SCHEMA
|
||||
self.schema = schemas.REQUIREMENTS_SCHEMA
|
||||
|
||||
def test_valid_requirements_data(self):
|
||||
requirements_data = [
|
||||
{"name": "test1", "versions": [">= 1.1.2", "<= 3"]},
|
||||
{"name": "test2", "versions": ["< 3", "> 1", ">= 4"]},
|
||||
{"name": "test3", "versions": ["= 3"]},
|
||||
{"name": "test4", "versions": ["= 3"]},
|
||||
{"name": "test4"}
|
||||
]
|
||||
requirements_data = {
|
||||
"packages": [
|
||||
{"name": "test1", "versions": [">= 1.1.2", "<= 3"]},
|
||||
{"name": "test2", "versions": ["< 3", "> 1", ">= 4"]},
|
||||
{"name": "test3", "versions": ["= 3"]},
|
||||
{"name": "test4", "versions": ["= 3"]},
|
||||
{"name": "test4"}
|
||||
],
|
||||
"repositories": [
|
||||
{"name": "repo1", "excludes": [{"name": "/a.+/"}]},
|
||||
{"name": "repo1", "excludes": [{"group": "debug"}]},
|
||||
{"name": "repo1"}
|
||||
],
|
||||
"all_mandatory": True,
|
||||
"options": ["localizations", "sources"]
|
||||
}
|
||||
self.assertNotRaises(
|
||||
jsonschema.ValidationError, jsonschema.validate, requirements_data,
|
||||
self.schema
|
||||
)
|
||||
|
||||
def test_validation_fail_for_required_properties(self):
|
||||
requirements_data = [
|
||||
[{"versions": ["< 3", "> 1"]}]
|
||||
def test_validation_fail_if_missed_required_properties(self):
|
||||
test_data = [
|
||||
{},
|
||||
{"packages": [{"version": "=2.0.0"}]},
|
||||
{"repositories": [{"excludes": [{"name": "test"}]}]},
|
||||
]
|
||||
for data in requirements_data:
|
||||
self.assertRaisesRegexp(
|
||||
jsonschema.ValidationError,
|
||||
"is a required property",
|
||||
jsonschema.validate, data, self.schema
|
||||
for data in test_data:
|
||||
self.assertRaises(
|
||||
jsonschema.ValidationError, jsonschema.validate,
|
||||
data, self.schema
|
||||
)
|
||||
|
||||
def test_validation_fail_if_name_is_invalid(self):
|
||||
requirements_data = [
|
||||
{"name": 123, "versions": [">= 1.1.2", "<= 3"]},
|
||||
]
|
||||
self.assertRaisesRegexp(
|
||||
jsonschema.ValidationError, "123 is not of type 'string'",
|
||||
jsonschema.validate, requirements_data, self.schema
|
||||
)
|
||||
|
||||
def test_validation_fail_if_versions_not_array(self):
|
||||
requirements_data = [
|
||||
{"name": "test1", "versions": 123}
|
||||
]
|
||||
self.assertRaisesRegexp(
|
||||
jsonschema.ValidationError, "123 is not of type 'array'",
|
||||
jsonschema.validate, requirements_data,
|
||||
self.schema
|
||||
)
|
||||
|
||||
def test_validation_fail_if_versions_not_string(self):
|
||||
requirements_data = [
|
||||
{"name": "test1", "versions": [123]}
|
||||
]
|
||||
self.assertRaisesRegexp(
|
||||
jsonschema.ValidationError, "123 is not of type 'string'",
|
||||
jsonschema.validate, requirements_data,
|
||||
self.schema
|
||||
)
|
||||
|
||||
def test_validation_fail_if_versions_not_match(self):
|
||||
versions = [
|
||||
["1.1.2"], # relational operator
|
||||
@ -240,14 +222,15 @@ class TestPackagesSchema(base.TestCase):
|
||||
for version in versions:
|
||||
self.assertRaisesRegexp(
|
||||
jsonschema.ValidationError, "does not match",
|
||||
jsonschema.validate, version,
|
||||
self.schema['items']['properties']['versions']
|
||||
jsonschema.validate,
|
||||
{"packages": [{"name": "test", "versions": version}]},
|
||||
self.schema
|
||||
)
|
||||
|
||||
|
||||
class TestPackageFilesSchema(base.TestCase):
|
||||
def setUp(self):
|
||||
self.schema = PACKAGE_FILES_SCHEMA
|
||||
self.schema = schemas.PACKAGE_FILES_SCHEMA
|
||||
|
||||
def test_valid_file_urls(self):
|
||||
file_urls = [
|
||||
|
Loading…
Reference in New Issue
Block a user