From d65d80efdc11641336160e96d366abb8d33de537 Mon Sep 17 00:00:00 2001 From: Federico Ressi Date: Fri, 22 Oct 2021 14:40:42 +0200 Subject: [PATCH] Add podman 1.6.0 source files and update requirements New requirements: - podman - python-dateutil>=2.8.0 - varlink Change-Id: I7c5ec418fd8f9c7d0d36fce033fe588a59e36f5d --- extra-requirements.txt | 3 +- requirements.txt | 2 + test-requirements.txt | 1 - tobiko/podman/_client.py | 6 +- tobiko/podman/_podman1/LICENSE | 201 +++++++++++++++ tobiko/podman/_podman1/README.md | 98 ++++++++ tobiko/podman/_podman1/__init__.py | 33 +++ tobiko/podman/_podman1/client.py | 221 ++++++++++++++++ tobiko/podman/_podman1/libs/__init__.py | 96 +++++++ .../_podman1/libs/_containers_attach.py | 81 ++++++ .../podman/_podman1/libs/_containers_start.py | 86 +++++++ tobiko/podman/_podman1/libs/containers.py | 237 ++++++++++++++++++ tobiko/podman/_podman1/libs/errors.py | 86 +++++++ tobiko/podman/_podman1/libs/images.py | 190 ++++++++++++++ tobiko/podman/_podman1/libs/pods.py | 160 ++++++++++++ tobiko/podman/_podman1/libs/system.py | 43 ++++ tobiko/podman/_podman1/libs/tunnel.py | 203 +++++++++++++++ tobiko/podman/_podman1/requirements.txt | 5 + tools/ensure_podman1.py | 45 ---- tools/install.py | 35 --- tox.ini | 4 +- 21 files changed, 1748 insertions(+), 88 deletions(-) create mode 100644 tobiko/podman/_podman1/LICENSE create mode 100644 tobiko/podman/_podman1/README.md create mode 100644 tobiko/podman/_podman1/__init__.py create mode 100644 tobiko/podman/_podman1/client.py create mode 100644 tobiko/podman/_podman1/libs/__init__.py create mode 100644 tobiko/podman/_podman1/libs/_containers_attach.py create mode 100644 tobiko/podman/_podman1/libs/_containers_start.py create mode 100644 tobiko/podman/_podman1/libs/containers.py create mode 100644 tobiko/podman/_podman1/libs/errors.py create mode 100644 tobiko/podman/_podman1/libs/images.py create mode 100644 tobiko/podman/_podman1/libs/pods.py create mode 100644 tobiko/podman/_podman1/libs/system.py create mode 100644 tobiko/podman/_podman1/libs/tunnel.py create mode 100644 tobiko/podman/_podman1/requirements.txt delete mode 100755 tools/ensure_podman1.py diff --git a/extra-requirements.txt b/extra-requirements.txt index 8f409b2b4..627d97705 100644 --- a/extra-requirements.txt +++ b/extra-requirements.txt @@ -3,7 +3,8 @@ ansi2html # LGPLv3+ dpkt # BSD pandas # BSD -podman-py # Apache-2.0 +podman # Apache-2.0 pytest-cov # MIT pytest-rerunfailures # MPL-2.0 pytest-timeout # MIT +varlink # Apache-2.0 diff --git a/requirements.txt b/requirements.txt index c117ab8ed..6325ce3b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,8 @@ oslo.config>=8.4.0 # Apache-2.0 oslo.log>=4.4.0 # Apache-2.0 paramiko>=2.7.2 # LGPLv2.1 pbr>=5.5.1 # Apache-2.0 +psutil>=5.8.0 # BSD +python-dateutil>=2.8.0 # Apache-2.0 python-glanceclient>=3.2.2 # Apache-2.0 python-heatclient>=2.3.0 # Apache-2.0 python-ironicclient>=4.6.1 # Apache-2.0 diff --git a/test-requirements.txt b/test-requirements.txt index 8d570f462..54ddf0040 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,6 @@ # Test cases requirements mock>=3.0.5 # BSD -psutil>=5.8.0 # BSD pytest>=6.2.1 # MIT pytest-html>=3.1.1 # MPL-2.0 pytest-xdist[psutil]>=2.2.0 # MIT diff --git a/tobiko/podman/_client.py b/tobiko/podman/_client.py index 1336058e2..756ff353b 100644 --- a/tobiko/podman/_client.py +++ b/tobiko/podman/_client.py @@ -11,11 +11,11 @@ import os from oslo_log import log import podman -import podman1 import tobiko from tobiko.podman import _exception +from tobiko.podman import _podman1 from tobiko.podman import _shell from tobiko.shell import ssh from tobiko.shell import sh @@ -37,7 +37,7 @@ def list_podman_containers(client=None, **kwargs): PODMAN_CLIENT_CLASSES = \ - podman1.Client, podman.PodmanClient # pylint: disable=E1101 + _podman1.Client, podman.PodmanClient # pylint: disable=E1101 def podman_client(obj=None): @@ -160,7 +160,7 @@ class PodmanClientFixture(tobiko.SharedFixture): LOG.info('container_client is online') else: - client = podman1.Client( # pylint: disable=E1101 + client = _podman1.Client( # pylint: disable=E1101 uri=podman_remote_socket_uri, remote_uri=remote_uri, identity_file='~/.ssh/id_rsa') diff --git a/tobiko/podman/_podman1/LICENSE b/tobiko/podman/_podman1/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/tobiko/podman/_podman1/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tobiko/podman/_podman1/README.md b/tobiko/podman/_podman1/README.md new file mode 100644 index 000000000..525d7295b --- /dev/null +++ b/tobiko/podman/_podman1/README.md @@ -0,0 +1,98 @@ +# podman - pythonic library for working with varlink interface to Podman + +[![Build Status](https://travis-ci.org/containers/python-podman.svg?branch=master)](https://travis-ci.org/containers/python-podman) +![PyPI](https://img.shields.io/pypi/v/podman.svg) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/podman.svg) +![PyPI - Status](https://img.shields.io/pypi/status/podman.svg) + +## Status: Active Development + +See [libpod](https://github.com/containers/python-podman) + +## Overview + +Python podman library. + +Provide a stable API to call into. + +## Releases + +### Requirements + +* Python 3.5+ +* OpenSSH 6.7+ +* Python dependencies in requirements.txt + +### Install + +#### From pypi + +Install `python-podman` to the standard location for third-party +Python modules: + +```sh +python3 -m pip install podman +``` + +To use this method on Unix/Linux system you need to have permission to write +to the standard third-party module directory. + +Else, you can install the latest version of python-podman published on +pypi to the Python user install directory for your platform. +Typically ~/.local/. ([See the Python documentation for site.USER_BASE for full +details.](https://pip.pypa.io/en/stable/user_guide/#user-installs)) +You can install like this by using the `--user` option: + +```sh +python3 -m pip install --user podman +``` + +This method can be useful in many situations, for example, +on a Unix system you might not have permission to write to the +standard third-party module directory. Or you might wish to try out a module +before making it a standard part of your local Python installation. +This is especially true when upgrading a distribution already present: you want +to make sure your existing base of scripts still works with the new version +before actually upgrading. + +For further reading about how python installation works [you can read +this documentation](https://docs.python.org/3/install/index.html#how-installation-works). + +#### By building from source + +To build the podman egg and install as user: + +```sh +cd ~/python-podman +python3 setup.py clean -a && python3 setup.py sdist bdist +python3 setup.py install --user +``` + +## Code snippets/examples: + +### Show images in storage + +```python +import podman + +with podman.Client() as client: + list(map(print, client.images.list())) +``` + +### Show containers created since midnight + +```python +from datetime import datetime, time, timezone + +import podman + +midnight = datetime.combine(datetime.today(), time.min, tzinfo=timezone.utc) + +with podman.Client() as client: + for c in client.containers.list(): + created_at = podman.datetime_parse(c.createdat) + + if created_at > midnight: + print('Container {}: image: {} created at: {}'.format( + c.id[:12], c.image[:32], podman.datetime_format(created_at))) +``` diff --git a/tobiko/podman/_podman1/__init__.py b/tobiko/podman/_podman1/__init__.py new file mode 100644 index 000000000..7da13fdaa --- /dev/null +++ b/tobiko/podman/_podman1/__init__.py @@ -0,0 +1,33 @@ +"""A client for communicating with a Podman server.""" + +from __future__ import absolute_import + +from pbr.version import VersionInfo + +from .client import Client +from .libs import FoldedString, datetime_format, datetime_parse +from .libs.errors import (ContainerNotFound, ErrorOccurred, ImageNotFound, + InvalidState, NoContainerRunning, NoContainersInPod, + PodContainerError, PodmanError, PodNotFound) + +assert FoldedString + +try: + __version__ = VersionInfo("podman") +except Exception: # pylint: disable=broad-except + __version__ = '0.0.0' + +__all__ = [ + 'Client', + 'ContainerNotFound', + 'datetime_format', + 'datetime_parse', + 'ErrorOccurred', + 'ImageNotFound', + 'InvalidState', + 'NoContainerRunning', + 'NoContainersInPod', + 'PodContainerError', + 'PodmanError', + 'PodNotFound', +] diff --git a/tobiko/podman/_podman1/client.py b/tobiko/podman/_podman1/client.py new file mode 100644 index 000000000..67ee1b3d4 --- /dev/null +++ b/tobiko/podman/_podman1/client.py @@ -0,0 +1,221 @@ +"""A client for communicating with a Podman varlink service.""" + +from __future__ import absolute_import + +import errno +import logging +import os +from urllib.parse import urlparse + +from varlink import Client as VarlinkClient +from varlink import VarlinkError + +from .libs import cached_property +from .libs.containers import Containers +from .libs.errors import error_factory +from .libs.images import Images +from .libs.system import System +from .libs.tunnel import Context, Portal, Tunnel +from .libs.pods import Pods + + +class BaseClient(): + """Context manager for API workers to access varlink.""" + + def __init__(self, context): + """Construct Client.""" + self._client = None + self._iface = None + self._context = context + + def __call__(self): + """Support being called for old API.""" + return self + + @classmethod + def factory(cls, + uri=None, + interface='io.podman', + *_args, + **kwargs): + """Construct a Client based on input.""" + # pylint: disable=keyword-arg-before-vararg + log_level = os.environ.get('LOG_LEVEL') + if log_level is not None: + logging.basicConfig(level=logging.getLevelName(log_level.upper())) + + if uri is None: + raise ValueError('uri is required and cannot be None') + if interface is None: + raise ValueError('interface is required and cannot be None') + + unsupported = set(kwargs.keys()).difference( + ('uri', 'interface', 'remote_uri', 'identity_file', + 'ignore_hosts', 'known_hosts')) + if unsupported: + raise ValueError('Unknown keyword arguments: {}'.format( + ', '.join(unsupported))) + + local_path = urlparse(uri).path + if local_path == '': + raise ValueError('path is required for uri,' + ' expected format "unix://path_to_socket"') + + if kwargs.get('remote_uri') is None: + return LocalClient(Context(uri, interface)) + + required = ('{} is required, expected format' + ' "ssh://user@hostname[:port]/path_to_socket".') + + # Remote access requires the full tuple of information + if kwargs.get('remote_uri') is None: + raise ValueError(required.format('remote_uri')) + + remote = urlparse(kwargs['remote_uri']) + if remote.username is None: + raise ValueError(required.format('username')) + if remote.path == '': + raise ValueError(required.format('path')) + if remote.hostname is None: + raise ValueError(required.format('hostname')) + + return RemoteClient( + Context( + uri, + interface, + local_path, + remote.path, + remote.username, + remote.hostname, + remote.port, + kwargs.get('identity_file'), + kwargs.get('ignore_hosts'), + kwargs.get('known_hosts'), + )) + + +class LocalClient(BaseClient): + """Context manager for API workers to access varlink.""" + + def __enter__(self): + """Enter context for LocalClient.""" + self._client = VarlinkClient(address=self._context.uri) + self._iface = self._client.open(self._context.interface) + return self._iface + + def __exit__(self, e_type, e, e_traceback): + """Cleanup context for LocalClient.""" + if hasattr(self._client, 'close'): + # pylint: disable=no-member + self._client.close() + self._iface.close() + + if isinstance(e, VarlinkError): + raise error_factory(e) + + +class RemoteClient(BaseClient): + """Context manager for API workers to access remote varlink.""" + + def __init__(self, context): + """Construct RemoteCLient.""" + super().__init__(context) + self._portal = Portal() + + def __enter__(self): + """Context manager for API workers to access varlink.""" + tunnel = self._portal.get(self._context.uri) + if tunnel is None: + tunnel = Tunnel(self._context).bore() + self._portal[self._context.uri] = tunnel + + try: + self._client = VarlinkClient(address=self._context.uri) + self._iface = self._client.open(self._context.interface) + return self._iface + except Exception: + tunnel.close() + raise + + def __exit__(self, e_type, e, e_traceback): + """Cleanup context for RemoteClient.""" + if hasattr(self._client, 'close'): + # pylint: disable=no-member + self._client.close() + self._iface.close() + + # set timer to shutdown ssh tunnel + # self._portal.get(self._context.uri).close() + if isinstance(e, VarlinkError): + raise error_factory(e) + + +class Client(): + """A client for communicating with a Podman varlink service. + + Example: + + >>> import podman + >>> c = podman.Client() + >>> c.system.versions + + Example remote podman: + + >>> import podman + >>> c = podman.Client(uri='unix:/tmp/podman.sock', + remote_uri='ssh://user@host/run/podman/io.podman', + identity_file='~/.ssh/id_rsa') + """ + + def __init__(self, + uri='unix:/run/podman/io.podman', + interface='io.podman', + **kwargs): + """Construct a podman varlink Client. + + uri from default systemd unit file. + interface from io.podman.varlink, do not change unless + you are a varlink guru. + """ + self._client = BaseClient.factory(uri, interface, **kwargs) + + address = "{}-{}".format(uri, interface) + # Quick validation of connection data provided + try: + if not System(self._client).ping(): + raise ConnectionRefusedError( + errno.ECONNREFUSED, + ('Failed varlink connection "{}"').format(address)) + except FileNotFoundError as ex: + raise ConnectionError( + errno.ECONNREFUSED, + ('Failed varlink connection "{}".' + ' Is podman socket or service running?' + ).format(address)) from ex + + def __enter__(self): + """Return `self` upon entering the runtime context.""" + return self + + def __exit__(self, exc_type, exc_value, traceback): + """Raise any exception triggered within the runtime context.""" + + @cached_property + def system(self): + """Manage system model for podman.""" + return System(self._client) + + @cached_property + def images(self): + """Manage images model for libpod.""" + return Images(self._client) + + @cached_property + def containers(self): + """Manage containers model for libpod.""" + return Containers(self._client) + + @cached_property + def pods(self): + """Manage pods model for libpod.""" + return Pods(self._client) diff --git a/tobiko/podman/_podman1/libs/__init__.py b/tobiko/podman/_podman1/libs/__init__.py new file mode 100644 index 000000000..bfae26573 --- /dev/null +++ b/tobiko/podman/_podman1/libs/__init__.py @@ -0,0 +1,96 @@ +"""Support files for podman API implementation.""" + +from __future__ import absolute_import + +import collections +import datetime +import functools + +from dateutil.parser import parse as dateutil_parse + +__all__ = [ + 'cached_property', + 'datetime_format', + 'datetime_parse', + 'flatten', + 'fold_keys', +] + + +def cached_property(fn): + """Decorate property to cache return value.""" + return property(functools.lru_cache(maxsize=8)(fn)) + + +class ConfigDict(collections.UserDict): + """Silently ignore None values, only take key once.""" + + def __init__(self, **kwargs): + """Construct dictionary.""" + super().__init__(kwargs) + + def __setitem__(self, key, value): + """Store unique, not None values.""" + if value is None: + return + + if super().__contains__(key): + return + + super().__setitem__(key, value) + + +class FoldedString(collections.UserString): + """Foldcase sequences value.""" + + def __init__(self, seq): + super().__init__(seq) + self.data.casefold() + + +def fold_keys(): # noqa: D202 + """Fold case of dictionary keys.""" + + @functools.wraps(fold_keys) + def wrapped(mapping): + """Fold case of dictionary keys.""" + return {k.casefold(): v for (k, v) in mapping.items()} + + return wrapped + + +def datetime_parse(string): + """Convert timestamps to datetime. + + tzinfo aware, if provided. + """ + return dateutil_parse(string.upper(), fuzzy=True) + + +def datetime_format(dt): + """Format datetime in consistent style.""" + if isinstance(dt, str): + return datetime_parse(dt).isoformat() + + if isinstance(dt, datetime.datetime): + return dt.isoformat() + + raise ValueError('Unable to format {}. Type {} not supported.'.format( + dt, type(dt))) + + +def flatten(list_, ltypes=(list, tuple)): + """Flatten lists of list into a list.""" + ltype = type(list_) + list_ = list(list_) + i = 0 + while i < len(list_): + while isinstance(list_[i], ltypes): + if not list_[i]: + list_.pop(i) + i -= 1 + break + else: + list_[i:i + 1] = list_[i] + i += 1 + return ltype(list_) diff --git a/tobiko/podman/_podman1/libs/_containers_attach.py b/tobiko/podman/_podman1/libs/_containers_attach.py new file mode 100644 index 000000000..4372521ec --- /dev/null +++ b/tobiko/podman/_podman1/libs/_containers_attach.py @@ -0,0 +1,81 @@ +"""Exported method Container.attach().""" + +from __future__ import absolute_import + +import collections +import fcntl +import logging +import struct +import sys +import termios + + +class Mixin: + """Publish attach() for inclusion in Container class.""" + + def attach(self, eot=4, stdin=None, stdout=None): + """Attach to container's PID1 stdin and stdout. + + stderr is ignored. + PseudoTTY work is done in start(). + """ + if stdin is None: + stdin = sys.stdin.fileno() + elif hasattr(stdin, 'fileno'): + stdin = stdin.fileno() + + if stdout is None: + stdout = sys.stdout.fileno() + elif hasattr(stdout, 'fileno'): + stdout = stdout.fileno() + + with self._client() as podman: + attach = podman.GetAttachSockets(self._id) + + # This is the UDS where all the IO goes + io_socket = attach['sockets']['io_socket'] + assert len(io_socket) <= 107,\ + 'Path length for sockets too long. {} > 107'.format( + len(io_socket) + ) + + # This is the control socket where resizing events are sent to conmon + # attach['sockets']['control_socket'] + self.pseudo_tty = collections.namedtuple( + 'PseudoTTY', + ['stdin', 'stdout', 'io_socket', 'control_socket', 'eot'])( + stdin, + stdout, + attach['sockets']['io_socket'], + attach['sockets']['control_socket'], + eot, + ) + + @property + def resize_handler(self): + """Send the new window size to conmon.""" + + def wrapped(signum, frame): # pylint: disable=unused-argument + packed = fcntl.ioctl(self.pseudo_tty.stdout, termios.TIOCGWINSZ, + struct.pack('HHHH', 0, 0, 0, 0)) + rows, cols, _, _ = struct.unpack('HHHH', packed) + logging.debug('Resize window(%dx%d) using %s', rows, cols, + self.pseudo_tty.control_socket) + + # TODO: Need some kind of timeout in case pipe is blocked + with open(self.pseudo_tty.control_socket, 'w') as skt: + # send conmon window resize message + skt.write('1 {} {}\n'.format(rows, cols)) + + return wrapped + + @property + def log_handler(self): + """Send command to reopen log to conmon.""" + + def wrapped(signum, frame): # pylint: disable=unused-argument + with open(self.pseudo_tty.control_socket, 'w') as skt: + # send conmon reopen log message + skt.write('2\n') + + return wrapped diff --git a/tobiko/podman/_podman1/libs/_containers_start.py b/tobiko/podman/_podman1/libs/_containers_start.py new file mode 100644 index 000000000..ec4bf6228 --- /dev/null +++ b/tobiko/podman/_podman1/libs/_containers_start.py @@ -0,0 +1,86 @@ +"""Exported method Container.start().""" + +from __future__ import absolute_import + +import logging +import os +import select +import signal +import socket +import sys +import termios +import tty + +CONMON_BUFSZ = 8192 + + +class Mixin: + """Publish start() for inclusion in Container class.""" + + def start(self): + """Start container, return container on success. + + Will block if container has been detached. + """ + # pylint: disable=protected-access + with self._client() as podman: + logging.debug('Starting Container "%s"', self._id) + results = podman.StartContainer(self._id) + logging.debug('Started Container "%s"', results['container']) + + if not hasattr(self, 'pseudo_tty') or self.pseudo_tty is None: + return self._refresh(podman) + + logging.debug('Setting up PseudoTTY for Container "%s"', + results['container']) + + try: + # save off the old settings for terminal + tcoldattr = termios.tcgetattr(self.pseudo_tty.stdin) + tty.setraw(self.pseudo_tty.stdin) + + # initialize container's window size + self.resize_handler(None, sys._getframe(0)) + + # catch any resizing events and send the resize info + # to the control fifo "socket" + signal.signal(signal.SIGWINCH, self.resize_handler) + + except termios.error: + tcoldattr = None + + try: + # TODO: Is socket.SOCK_SEQPACKET supported in Windows? + with socket.socket(socket.AF_UNIX, + socket.SOCK_SEQPACKET) as skt: + # Prepare socket for use with conmon/container + skt.connect(self.pseudo_tty.io_socket) + + sources = [skt, self.pseudo_tty.stdin] + while sources: + logging.debug('Waiting on sources: %s', sources) + readable, _, _ = select.select(sources, [], []) + + if skt in readable: + data = skt.recv(CONMON_BUFSZ) + if data: + # Remove source marker when writing + os.write(self.pseudo_tty.stdout, data[1:]) + else: + sources.remove(skt) + + if self.pseudo_tty.stdin in readable: + data = os.read(self.pseudo_tty.stdin, CONMON_BUFSZ) + if data: + skt.sendall(data) + + if self.pseudo_tty.eot in data: + sources.clear() + else: + sources.remove(self.pseudo_tty.stdin) + finally: + if tcoldattr: + termios.tcsetattr(self.pseudo_tty.stdin, termios.TCSADRAIN, + tcoldattr) + signal.signal(signal.SIGWINCH, signal.SIG_DFL) + return self._refresh(podman) diff --git a/tobiko/podman/_podman1/libs/containers.py b/tobiko/podman/_podman1/libs/containers.py new file mode 100644 index 000000000..f4efebd71 --- /dev/null +++ b/tobiko/podman/_podman1/libs/containers.py @@ -0,0 +1,237 @@ +"""Models for manipulating containers and storage.""" + +from __future__ import absolute_import + +import collections +import getpass +import json +import logging +import signal +import time + +from . import fold_keys +from ._containers_attach import Mixin as AttachMixin +from ._containers_start import Mixin as StartMixin + + +class Container(AttachMixin, StartMixin, collections.UserDict): + """Model for a container.""" + + def __init__(self, client, ident, data, refresh=True): + """Construct Container Model.""" + super(Container, self).__init__(data) + self._client = client + self._id = ident + + if refresh: + with client() as podman: + self._refresh(podman) + else: + for k, v in self.data.items(): + setattr(self, k, v) + if 'containerrunning' in self.data: + setattr(self, 'running', self.data['containerrunning']) + self.data['running'] = self.data['containerrunning'] + + assert self._id == data['id'],\ + 'Requested container id({}) does not match store id({})'.format( + self._id, data['id'] + ) + + def _refresh(self, podman, tries=1): + try: + ctnr = podman.GetContainer(self._id) + except BrokenPipeError: + logging.debug('Failed GetContainer(%s) try %d/3', self._id, tries) + if tries > 3: + raise + else: + with self._client() as pman: + self._refresh(pman, tries + 1) + else: + super().update(ctnr['container']) + + for k, v in self.data.items(): + setattr(self, k, v) + if 'containerrunning' in self.data: + setattr(self, 'running', self.data['containerrunning']) + self.data['running'] = self.data['containerrunning'] + + return self + + def refresh(self): + """Refresh status fields for this container.""" + with self._client() as podman: + return self._refresh(podman) + + def processes(self): + """Show processes running in container.""" + with self._client() as podman: + results = podman.ListContainerProcesses(self._id) + yield from results['container'] + + def changes(self): + """Retrieve container changes.""" + with self._client() as podman: + results = podman.ListContainerChanges(self._id) + return results['container'] + + def kill(self, sig=signal.SIGTERM, wait=25): + """Send signal to container. + + default signal is signal.SIGTERM. + wait n of seconds, 0 waits forever. + """ + with self._client() as podman: + podman.KillContainer(self._id, sig) + timeout = time.time() + wait + while True: + self._refresh(podman) + if self.status != 'running': # pylint: disable=no-member + return self + + if wait and timeout < time.time(): + raise TimeoutError() + + time.sleep(0.5) + + def inspect(self): + """Retrieve details about containers.""" + with self._client() as podman: + results = podman.InspectContainer(self._id) + obj = json.loads(results['container'], object_hook=fold_keys()) + return collections.namedtuple('ContainerInspect', obj.keys())(**obj) + + def export(self, target): + """Export container from store to tarball. + + TODO: should there be a compress option, like images? + """ + with self._client() as podman: + results = podman.ExportContainer(self._id, target) + return results['tarfile'] + + def commit(self, image_name, **kwargs): + """Create image from container. + + Keyword arguments: + author -- change image's author + message -- change image's message, docker format only. + pause -- pause container during commit + change -- Additional properties to change + + Change examples: + CMD=/usr/bin/zsh + ENTRYPOINT=/bin/sh date + ENV=TEST=test_containers.TestContainers.test_commit + EXPOSE=8888/tcp + LABEL=unittest=test_commit + USER=bozo:circus + VOLUME=/data + WORKDIR=/data/application + + All changes overwrite existing values. + See inspect() to obtain current settings. + """ + author = kwargs.get('author', None) or getpass.getuser() + change = kwargs.get('change', None) or [] + message = kwargs.get('message', None) or '' + pause = kwargs.get('pause', None) or True + + for c in change: + if c.startswith('LABEL=') and c.count('=') < 2: + raise ValueError( + 'LABEL should have the format: LABEL=label=value, not {}'. + format(c)) + + with self._client() as podman: + results = podman.Commit(self._id, image_name, change, author, + message, pause) + return results['reply']['id'] + + def stop(self, timeout=25): + """Stop container, return id on success.""" + with self._client() as podman: + podman.StopContainer(self._id, timeout) + return self._refresh(podman) + + def remove(self, force=False): + """Remove container, return id on success. + + force=True, stop running container. + """ + with self._client() as podman: + results = podman.RemoveContainer(self._id, force) + return results['container'] + + def restart(self, timeout=25): + """Restart container with timeout, return id on success.""" + with self._client() as podman: + podman.RestartContainer(self._id, timeout) + return self._refresh(podman) + + def pause(self): + """Pause container, return id on success.""" + with self._client() as podman: + podman.PauseContainer(self._id) + return self._refresh(podman) + + def unpause(self): + """Unpause container, return id on success.""" + with self._client() as podman: + podman.UnpauseContainer(self._id) + return self._refresh(podman) + + def update_container(self, *args, **kwargs): \ + # pylint: disable=unused-argument + """TODO: Update container..., return id on success.""" + with self._client() as podman: + podman.UpdateContainer() + return self._refresh(podman) + + def wait(self): + """Wait for container to finish, return 'returncode'.""" + with self._client() as podman: + results = podman.WaitContainer(self._id) + return int(results['exitcode']) + + def stats(self): + """Retrieve resource stats from the container.""" + with self._client() as podman: + results = podman.GetContainerStats(self._id) + obj = results['container'] + return collections.namedtuple('StatDetail', obj.keys())(**obj) + + def logs(self, *args, **kwargs): # pylint: disable=unused-argument + """Retrieve container logs.""" + with self._client() as podman: + results = podman.GetContainerLogs(self._id) + yield from results['container'] + + +class Containers(): + """Model for Containers collection.""" + + def __init__(self, client): + """Construct model for Containers collection.""" + self._client = client + + def list(self): + """List of containers in the container store.""" + with self._client() as podman: + results = podman.ListContainers() + for cntr in results['containers']: + yield Container(self._client, cntr['id'], cntr, refresh=False) + + def delete_stopped(self): + """Delete all stopped containers.""" + with self._client() as podman: + results = podman.DeleteStoppedContainers() + return results['containers'] + + def get(self, id_): + """Retrieve container details from store.""" + with self._client() as podman: + cntr = podman.GetContainer(id_) + return Container(self._client, cntr['container']['id'], + cntr['container']) diff --git a/tobiko/podman/_podman1/libs/errors.py b/tobiko/podman/_podman1/libs/errors.py new file mode 100644 index 000000000..8fd04b3cf --- /dev/null +++ b/tobiko/podman/_podman1/libs/errors.py @@ -0,0 +1,86 @@ +"""Error classes and wrappers for VarlinkError.""" + +from __future__ import absolute_import + +from varlink import VarlinkError + + +class VarlinkErrorProxy(VarlinkError): + """Class to Proxy VarlinkError methods.""" + + def __init__(self, message, namespaced=False): + """Construct proxy from Exception.""" + super().__init__(message.as_dict(), namespaced) + self._message = message + self.__module__ = 'libpod' + + def __getattr__(self, method): + """Return attribute from proxied Exception.""" + if hasattr(self._message, method): + return getattr(self._message, method) + + try: + return self._message.parameters()[method] + except KeyError as ex: + raise AttributeError( + 'No such attribute: {}'.format(method)) from ex + + +class ContainerNotFound(VarlinkErrorProxy): + """Raised when Client cannot find requested container.""" + + +class ImageNotFound(VarlinkErrorProxy): + """Raised when Client cannot find requested image.""" + + +class PodNotFound(VarlinkErrorProxy): + """Raised when Client cannot find requested image.""" + + +class PodContainerError(VarlinkErrorProxy): + """Raised when a container fails requested pod operation.""" + + +class NoContainerRunning(VarlinkErrorProxy): + """Raised when no container is running in pod.""" + + +class NoContainersInPod(VarlinkErrorProxy): + """Raised when Client fails to connect to runtime.""" + + +class ErrorOccurred(VarlinkErrorProxy): + """Raised when an error occurs during the execution. + + See error() to see actual error text. + """ + + +class PodmanError(VarlinkErrorProxy): + """Raised when Client fails to connect to runtime.""" + + +class InvalidState(VarlinkErrorProxy): + """Raised when container is in invalid state for operation.""" + + +ERROR_MAP = { + 'io.podman.ContainerNotFound': ContainerNotFound, + 'io.podman.ErrorOccurred': ErrorOccurred, + 'io.podman.ImageNotFound': ImageNotFound, + 'io.podman.InvalidState': InvalidState, + 'io.podman.NoContainerRunning': NoContainerRunning, + 'io.podman.NoContainersInPod': NoContainersInPod, + 'io.podman.PodContainerError': PodContainerError, + 'io.podman.PodNotFound': PodNotFound, + 'io.podman.RuntimeError': PodmanError, +} + + +def error_factory(exception): + """Map Exceptions to a discrete type.""" + try: + return ERROR_MAP[exception.error()](exception) + except KeyError: + return exception diff --git a/tobiko/podman/_podman1/libs/images.py b/tobiko/podman/_podman1/libs/images.py new file mode 100644 index 000000000..e7ed928ed --- /dev/null +++ b/tobiko/podman/_podman1/libs/images.py @@ -0,0 +1,190 @@ +"""Models for manipulating images in/to/from storage.""" + +from __future__ import absolute_import + +import collections +import copy +import json +import logging + +from . import ConfigDict, flatten, fold_keys +from .containers import Container + + +class Image(collections.UserDict): + """Model for an Image.""" + + def __init__(self, client, id, data): + """Construct Image Model.""" + # pylint: disable=redefined-builtin + + super().__init__(data) + for k, v in data.items(): + setattr(self, k, v) + + self._id = id + self._client = client + + assert self._id == data['id'],\ + 'Requested image id({}) does not match store id({})'.format( + self._id, data['id'] + ) + + @staticmethod + def _split_token(values=None, sep='='): + if not values: + return {} + return {k: v1 for k, v1 in (v0.split(sep, 1) for v0 in values)} + + def create(self, *_args, **kwargs): + """Create container from image. + + Pulls defaults from image.inspect() + """ + details = self.inspect() + + config = ConfigDict(image_id=self._id, **kwargs) + config['command'] = details.config.get('cmd') + config['env'] = self._split_token(details.config.get('env')) + config['image'] = copy.deepcopy(details.repotags[0]) + config['labels'] = copy.deepcopy(details.labels) + # TODO: Are these settings still required? + config['net_mode'] = 'bridge' + config['network'] = 'bridge' + config['args'] = flatten([config['image'], config['command']]) + + logging.debug('Image %s: create config: %s', self._id, config) + with self._client() as podman: + id_ = podman.CreateContainer(config)['container'] + cntr = podman.GetContainer(id_) + return Container(self._client, id_, cntr['container']) + + container = create + + def export(self, dest, compressed=False): + """Write image to dest, return id on success.""" + with self._client() as podman: + results = podman.ExportImage(self._id, dest, compressed) + return results['image'] + + def history(self): + """Retrieve image history.""" + with self._client() as podman: + for r in podman.HistoryImage(self._id)['history']: + yield collections.namedtuple('HistoryDetail', r.keys())(**r) + + def inspect(self): + """Retrieve details about image.""" + with self._client() as podman: + results = podman.InspectImage(self._id) + obj = json.loads(results['image'], object_hook=fold_keys()) + return collections.namedtuple('ImageInspect', obj.keys())(**obj) + + def push(self, + target, + compress=False, + manifest_format="", + remove_signatures=False, + sign_by=""): + """Copy image to target, return id on success.""" + with self._client() as podman: + results = podman.PushImage(self._id, target, compress, + manifest_format, remove_signatures, + sign_by) + return results['reply']['id'] + + def remove(self, force=False): + """Delete image, return id on success. + + force=True, stop any running containers using image. + """ + with self._client() as podman: + results = podman.RemoveImage(self._id, force) + return results['image'] + + def tag(self, tag): + """Tag image.""" + with self._client() as podman: + results = podman.TagImage(self._id, tag) + return results['image'] + + +class Images(): + """Model for Images collection.""" + + def __init__(self, client): + """Construct model for Images collection.""" + self._client = client + + def list(self): + """List all images in the libpod image store.""" + with self._client() as podman: + results = podman.ListImages() + for img in results['images']: + yield Image(self._client, img['id'], img) + + def build(self, dockerfile=None, tags=None, **kwargs): + """Build container from image. + + See podman-build.1.md for kwargs details. + """ + if dockerfile is None: + raise ValueError('"dockerfile" is a required argument.') + if not hasattr(dockerfile, '__iter__'): + raise ValueError('"dockerfile" is required to be an iter.') + + if tags is None: + raise ValueError('"tags" is a required argument.') + if not hasattr(tags, '__iter__'): + raise ValueError('"tags" is required to be an iter.') + + config = ConfigDict(dockerfile=dockerfile, tags=tags, **kwargs) + with self._client() as podman: + result = podman.BuildImage(config) + return self.get(result['image']['id']), \ + (line for line in result['image']['logs']) + + def delete_unused(self): + """Delete Images not associated with a container.""" + with self._client() as podman: + results = podman.DeleteUnusedImages() + return results['images'] + + def import_image(self, source, reference, message='', changes=None): + """Read image tarball from source and save in image store.""" + with self._client() as podman: + results = podman.ImportImage(source, reference, message, changes) + return results['image'] + + def pull(self, source): + """Copy image from registry to image store.""" + with self._client() as podman: + results = podman.PullImage(source) + return results['reply']['id'] + + def search(self, + id_, + limit=25, + is_official=None, + is_automated=None, + star_count=None): + """Search registries for id.""" + constraints = {} + + if is_official is not None: + constraints['is_official'] = is_official + if is_automated is not None: + constraints['is_automated'] = is_automated + if star_count is not None: + constraints['star_count'] = star_count + + with self._client() as podman: + results = podman.SearchImages(id_, limit, constraints) + for img in results['results']: + yield collections.namedtuple('ImageSearch', img.keys())(**img) + + def get(self, id_): + """Get Image from id.""" + with self._client() as podman: + result = podman.GetImage(id_) + return Image(self._client, result['image']['id'], result['image']) diff --git a/tobiko/podman/_podman1/libs/pods.py b/tobiko/podman/_podman1/libs/pods.py new file mode 100644 index 000000000..62ffd6d2f --- /dev/null +++ b/tobiko/podman/_podman1/libs/pods.py @@ -0,0 +1,160 @@ +"""Model for accessing details of Pods from podman service.""" + +from __future__ import absolute_import + +import collections +import json +import signal +import time + +from . import ConfigDict, FoldedString, fold_keys + + +class Pod(collections.UserDict): + """Model for a Pod.""" + + def __init__(self, client, ident, data): + """Construct Pod model.""" + super().__init__(data) + + self._ident = ident + self._client = client + + with client() as podman: + self._refresh(podman) + + def _refresh(self, podman): + pod = podman.GetPod(self._ident) + super().update(pod['pod']) + + for k, v in self.data.items(): + setattr(self, k, v) + return self + + def inspect(self): + """Retrieve details about pod.""" + with self._client() as podman: + results = podman.InspectPod(self._ident) + obj = json.loads(results['pod'], object_hook=fold_keys()) + obj['id'] = obj['config']['id'] + return collections.namedtuple('PodInspect', obj.keys())(**obj) + + def kill(self, signal_=signal.SIGTERM, wait=25): + """Send signal to all containers in pod. + + default signal is signal.SIGTERM. + wait n of seconds, 0 waits forever. + """ + with self._client() as podman: + podman.KillPod(self._ident, signal_) + timeout = time.time() + wait + while True: + # pylint: disable=maybe-no-member + self._refresh(podman) + running = FoldedString(self.status) + if running != 'running': + break + + if wait and timeout < time.time(): + raise TimeoutError() + + time.sleep(0.5) + return self + + def pause(self): + """Pause all containers in the pod.""" + with self._client() as podman: + podman.PausePod(self._ident) + return self._refresh(podman) + + def refresh(self): + """Refresh status fields for this pod.""" + with self._client() as podman: + return self._refresh(podman) + + def remove(self, force=False): + """Remove pod and its containers returning pod ident. + + force=True, stop any running container. + """ + with self._client() as podman: + results = podman.RemovePod(self._ident, force) + return results['pod'] + + def restart(self): + """Restart all containers in the pod.""" + with self._client() as podman: + podman.RestartPod(self._ident) + return self._refresh(podman) + + def stats(self): + """Stats on all containers in the pod.""" + with self._client() as podman: + results = podman.GetPodStats(self._ident) + for obj in results['containers']: + yield collections.namedtuple('ContainerStats', obj.keys())(**obj) + + def start(self): + """Start all containers in the pod.""" + with self._client() as podman: + podman.StartPod(self._ident) + return self._refresh(podman) + + def stop(self): + """Stop all containers in the pod.""" + with self._client() as podman: + podman.StopPod(self._ident) + return self._refresh(podman) + + def top(self): + """Display stats for all containers.""" + with self._client() as podman: + results = podman.TopPod(self._ident) + return results['pod'] + + def unpause(self): + """Unpause all containers in the pod.""" + with self._client() as podman: + podman.UnpausePod(self._ident) + return self._refresh(podman) + + +class Pods(): + """Model for accessing pods.""" + + def __init__(self, client): + """Construct pod model.""" + self._client = client + + def create(self, + ident=None, + cgroupparent=None, + labels=None, + share=None, + infra=False): + """Create a new empty pod.""" + config = ConfigDict( + name=ident, + cgroupParent=cgroupparent, + labels=labels, + share=share, + infra=infra, + ) + + with self._client() as podman: + result = podman.CreatePod(config) + details = podman.GetPod(result['pod']) + return Pod(self._client, result['pod'], details['pod']) + + def get(self, ident): + """Get Pod from ident.""" + with self._client() as podman: + result = podman.GetPod(ident) + return Pod(self._client, result['pod']['id'], result['pod']) + + def list(self): + """List all pods.""" + with self._client() as podman: + results = podman.ListPods() + for pod in results['pods']: + yield Pod(self._client, pod['id'], pod) diff --git a/tobiko/podman/_podman1/libs/system.py b/tobiko/podman/_podman1/libs/system.py new file mode 100644 index 000000000..9d7eaef3c --- /dev/null +++ b/tobiko/podman/_podman1/libs/system.py @@ -0,0 +1,43 @@ +"""Models for accessing details from varlink server.""" + +from __future__ import absolute_import + +import collections + +import pkg_resources + +from . import cached_property + + +class System(): + """Model for accessing system resources.""" + + def __init__(self, client): + """Construct system model.""" + self._client = client + + @cached_property + def versions(self): + """Access versions.""" + with self._client() as podman: + vers = podman.GetVersion() + + client = '0.0.0' + try: + client = pkg_resources.get_distribution('podman').version + except Exception: # pylint: disable=broad-except + pass + vers['client_version'] = client + return collections.namedtuple('Version', vers.keys())(**vers) + + def info(self): + """Return podman info.""" + with self._client() as podman: + info = podman.GetInfo()['info'] + return collections.namedtuple('Info', info.keys())(**info) + + def ping(self): + """Return True if server awake.""" + with self._client() as podman: + response = podman.GetVersion() + return 'version' in response diff --git a/tobiko/podman/_podman1/libs/tunnel.py b/tobiko/podman/_podman1/libs/tunnel.py new file mode 100644 index 000000000..b71425f9e --- /dev/null +++ b/tobiko/podman/_podman1/libs/tunnel.py @@ -0,0 +1,203 @@ +"""Cache for SSH tunnels.""" + +from __future__ import absolute_import + +import collections +import getpass +import logging +import os +import subprocess +import threading +import time +import weakref +from contextlib import suppress + +import psutil + +Context = collections.namedtuple('Context', ( + 'uri', + 'interface', + 'local_socket', + 'remote_socket', + 'username', + 'hostname', + 'port', + 'identity_file', + 'ignore_hosts', + 'known_hosts', +)) +Context.__new__.__defaults__ = (None, ) * len(Context._fields) # type: ignore + + +class Portal(collections.MutableMapping): + """Expiring container for tunnels.""" + + def __init__(self, sweap=25): + """Construct portal, reap tunnels every sweap seconds.""" + self.data = collections.OrderedDict() + self.sweap = sweap + self.ttl = sweap * 2 + self.lock = threading.RLock() + self._schedule_reaper() + + def __getitem__(self, key): + """Given uri return tunnel and update TTL.""" + with self.lock: + value, _ = self.data[key] + self.data[key] = (value, time.time() + self.ttl) + self.data.move_to_end(key) + return value + + def __setitem__(self, key, value): + """Store given tunnel keyed with uri.""" + if not isinstance(value, Tunnel): + raise ValueError('Portals only support Tunnels.') + + with self.lock: + self.data[key] = (value, time.time() + self.ttl) + self.data.move_to_end(key) + + def __delitem__(self, key): + """Remove and close tunnel from portal.""" + with self.lock: + value, _ = self.data[key] + del self.data[key] + value.close() + del value + + def __iter__(self): + """Iterate tunnels.""" + with self.lock: + values = list(self.data.values()) + + for tunnel, _ in values: + yield tunnel + + def __len__(self): + """Return number of tunnels in portal.""" + with self.lock: + return len(self.data) + + def _schedule_reaper(self): + timer = threading.Timer(interval=self.sweap, function=self.reap) + timer.setName('PortalReaper') + timer.setDaemon(True) + timer.start() + + def reap(self): + """Remove tunnels who's TTL has expired.""" + now = time.time() + with self.lock: + reaped_data = self.data.copy() + for entry in reaped_data.items(): + if entry[1][1] < now: + del self.data[entry[0]] + else: + # StopIteration as soon as possible + break + self._schedule_reaper() + + +class Tunnel(): + """SSH tunnel.""" + + def __init__(self, context): + """Construct Tunnel.""" + self.context = context + self._closed = True + + @property + def closed(self): + """Is tunnel closed.""" + return self._closed + + def bore(self): + """Create SSH tunnel from given context.""" + cmd = ['ssh', '-fNT'] + + if logging.getLogger().getEffectiveLevel() == logging.DEBUG: + cmd.append('-v') + else: + cmd.append('-q') + + if self.context.port: + cmd.extend(('-p', str(self.context.port))) + + cmd.extend(('-L', '{}:{}'.format(self.context.local_socket, + self.context.remote_socket))) + + if self.context.ignore_hosts: + cmd.extend(('-o', 'StrictHostKeyChecking=no', + '-o', 'UserKnownHostsFile=/dev/null')) + elif self.context.known_hosts: + cmd.extend(('-o', 'UserKnownHostsFile={known_hosts}'.format( + known_hosts=self.context.known_hosts))) + + if self.context.identity_file: + cmd.extend(('-i', self.context.identity_file)) + + cmd.append('{}@{}'.format(self.context.username, + self.context.hostname)) + + logging.debug('Opening tunnel "%s", cmd "%s"', self.context.uri, + ' '.join(cmd)) + + tunnel = subprocess.Popen(cmd, close_fds=True) + # The return value of Popen() has no long term value as that process + # has already exited by the time control is returned here. This is a + # side effect of the -f option. wait() will be called to clean up + # resources. + for _ in range(300): + # TODO: Make timeout configurable + if os.path.exists(self.context.local_socket) \ + or tunnel.returncode is not None: + break + with suppress(subprocess.TimeoutExpired): + # waiting for either socket to be created + # or first child to exit + tunnel.wait(0.5) + else: + raise TimeoutError( + 'Failed to create tunnel "{}", using: "{}"'.format( + self.context.uri, ' '.join(cmd))) + if tunnel.returncode is not None and tunnel.returncode != 0: + raise subprocess.CalledProcessError(tunnel.returncode, + ' '.join(cmd)) + tunnel.wait() + + self._closed = False + weakref.finalize(self, self.close) + return self + + def close(self): + """Close SSH tunnel.""" + logging.debug('Closing tunnel "%s"', self.context.uri) + + if self._closed: + return + + # Find all ssh instances for user with uri tunnel the hard way... + targets = [ + p + for p in psutil.process_iter(attrs=['name', 'username', 'cmdline']) + if p.info['username'] == getpass.getuser() + and p.info['name'] == 'ssh' + and self.context.local_socket in ' '.join(p.info['cmdline']) + ] # yapf: disable + + # ask nicely for ssh to quit, reap results + for proc in targets: + proc.terminate() + _, alive = psutil.wait_procs(targets, timeout=300) + + # kill off the uncooperative, then report any stragglers + for proc in alive: + proc.kill() + _, alive = psutil.wait_procs(targets, timeout=300) + + for proc in alive: + logging.info('process %d survived SIGKILL, giving up.', proc.pid) + + with suppress(OSError): + os.remove(self.context.local_socket) + self._closed = True diff --git a/tobiko/podman/_podman1/requirements.txt b/tobiko/podman/_podman1/requirements.txt new file mode 100644 index 000000000..dda3b05ac --- /dev/null +++ b/tobiko/podman/_podman1/requirements.txt @@ -0,0 +1,5 @@ +pbr +psutil +python-dateutil +setuptools>=39 +varlink diff --git a/tools/ensure_podman1.py b/tools/ensure_podman1.py deleted file mode 100755 index 4593c0f45..000000000 --- a/tools/ensure_podman1.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2018 Red Hat -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from __future__ import absolute_import - -import os -import sys - -TOP_DIR = os.path.realpath(os.path.dirname(os.path.dirname(__file__))) - -if TOP_DIR not in sys.path: - sys.path.insert(0, TOP_DIR) - -from tools import common # noqa -from tools import install # noqa - - -LOG = common.get_logger(__name__) - - -def main(): - common.setup_logging() - ensure_podman1() - - -def ensure_podman1(): - try: - import podman1 - except ImportError: - install.install_podman1() - - -if __name__ == '__main__': - main() diff --git a/tools/install.py b/tools/install.py index 5d5a7fe8c..490a9fb31 100755 --- a/tools/install.py +++ b/tools/install.py @@ -39,7 +39,6 @@ def main(): common.setup_logging() install_tox() install_bindeps() - install_podman1() install_tobiko() @@ -60,40 +59,6 @@ def install_tobiko(): pip_install(f"-e '{TOP_DIR}'") -def install_podman1(version='===1.6.0'): - pip_unisntall('podman') - - LOG.info(f"Installing Podman... (version: {version})") - - site_dirs = {os.path.dirname(os.path.realpath(site_dir)) - for site_dir in site.getsitepackages() - if os.path.isdir(site_dir)} - more_site_dirs = {os.path.join(site_dir, 'site-packages') - for site_dir in site_dirs - if os.path.isdir(os.path.join(site_dir, 'site-packages'))} - site_dirs.update(more_site_dirs) - LOG.debug(f"Site packages dirs: {site_dirs}") - - # Must ensure pre-existing podman directories are restored - # after installation - podman_dirs = [os.path.join(site_dir, 'podman') - for site_dir in sorted(site_dirs)] - LOG.debug(f"Possible podman directories: {podman_dirs}") - with common.stash_dir(*podman_dirs): - for podman_dir in podman_dirs: - assert not os.path.exists(podman_dir) - pip_install(f"'podman{version}'") - for podman_dir in podman_dirs: - if os.path.isdir(podman_dir): - # Rename podman directory to podman1 - os.rename(podman_dir, podman_dir + '1') - break - else: - raise RuntimeError("Podman directory not found!") - for podman_dir in podman_dirs: - assert not os.path.exists(podman_dir) - - def pip_install(args): LOG.debug(f"Installing packages: {args}...") common.execute_python(f"-m pip install {TOX_CONSTRAINTS} {args}", diff --git a/tox.ini b/tox.ini index ba61c569d..99291dabc 100644 --- a/tox.ini +++ b/tox.ini @@ -38,8 +38,6 @@ setenv = TOX_CONSTRAINTS = {env:TOX_CONSTRAINTS:-c{toxinidir}/upper-constraints.txt} TOX_EXTRA_REQUIREMENTS = {env:TOX_EXTRA_REQUIREMENTS:-r{toxinidir}/extra-requirements.txt} VIRTUAL_ENV = {envdir} -commands_pre = - {envpython} {toxinidir}/tools/ensure_podman1.py commands = {envpython} {toxinidir}/tools/run_tests.py {posargs:{env:RUN_TESTS_EXTRA_ARGS}} @@ -139,7 +137,7 @@ enable-extensions = H106,H203,H204,H205,H904 show-source = true exclude = ./.*,*lib/python*,build,dist,doc,*egg*,releasenotes,.venv,.tox application-import-names = tobiko -max-complexity = 10 +max-complexity = 11 import-order-style = pep8