From 31758b2400456714f6e90f9e58568aab25a56a95 Mon Sep 17 00:00:00 2001 From: Federico Ressi Date: Tue, 4 Jan 2022 12:08:09 +0100 Subject: [PATCH] Update Tobiko container - allow to use fedora 35 as base image - make base image configurable - update docker compose file - fix python code to satisfy linters with Python 3.10 - add linters, lower-constraints and fedora jobs Change-Id: I1a786191d2529fe1d14a47f06f76405490bfb11c --- Dockerfile | 73 +++++++++++++------ docker-compose.yml | 24 ++++++ tobiko/podman/_podman1/libs/tunnel.py | 4 +- .../functional/openstack/test_topology.py | 4 +- zuul.d/docker.yaml | 43 ++++++++++- 5 files changed, 119 insertions(+), 29 deletions(-) diff --git a/Dockerfile b/Dockerfile index b5807434f..4b038d214 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,43 @@ -FROM python:3.10 as base +ARG base_image=python + + +FROM python:3.10 as python + +ENV INSTALL_PACKAGES="apt install -y" +ENV CONSTRAINS_FILE=upper-constraints.txt + +RUN apt update && apt install -y iperf3 iputils-ping ncat + + +FROM python:3.8 as lower-constraints + +ENV INSTALL_PACKAGES="apt install -y" +ENV CONSTRAINS_FILE=lower-constraints.txt + +RUN apt update && apt install -y iperf3 iputils-ping ncat + + +FROM fedora:35 as fedora + +ENV CONSTRAINS_FILE=upper-constraints.txt +ENV INSTALL_PACKAGES="dnf install -y" + +RUN ${INSTALL_PACKAGES} iperf3 gcc python3-devel + + +FROM ${base_image} as base ENV TOBIKO_DIR=/tobiko ENV WHEEL_DIR=/wheel -ENV INSTALL_PACKAGES="apt install -y" - -# Install binary dependencies -RUN apt update && \ - ${INSTALL_PACKAGES} git - RUN python3 -m ensurepip --upgrade && \ python3 -m pip install --upgrade setuptools wheel FROM base as source -# Populate tobiko source dir -# RUN mkdir -p ${TOBIKO_DIR} +# Install binary dependencies +RUN ${INSTALL_PACKAGES} git ADD .gitignore \ extra-requirements.txt \ @@ -25,21 +46,18 @@ ADD .gitignore \ setup.cfg \ setup.py \ test-requirements.txt \ - upper-constraints.txt \ + ${CONSTRAINS_FILE} \ ${TOBIKO_DIR}/ -ADD .git ${TOBIKO_DIR}/.git/ ADD tobiko/ ${TOBIKO_DIR}/tobiko/ +ADD .git ${TOBIKO_DIR}/.git/ FROM source as build -# Install binary dependencies -# RUN ${INSTALL_PACKAGES} gcc python3-devel - # Build wheel files RUN python3 -m pip wheel -w ${WHEEL_DIR} \ - -c ${TOBIKO_DIR}/upper-constraints.txt \ + -c ${TOBIKO_DIR}/${CONSTRAINS_FILE} \ -r ${TOBIKO_DIR}/requirements.txt \ -r ${TOBIKO_DIR}/test-requirements.txt \ -r ${TOBIKO_DIR}/extra-requirements.txt \ @@ -56,9 +74,6 @@ RUN python3 -m pip install ${WHEEL_DIR}/*.whl FROM source as tobiko -# Install packages -RUN ${INSTALL_PACKAGES} iperf3 iputils-ping ncat - # Run tests variables ENV PYTHONWARNINGS=ignore::Warning ENV OS_TEST_PATH=${TOBIKO_DIR}/tobiko/tests/unit @@ -70,11 +85,27 @@ ENV TOBIKO_PREVENT_CREATE=false RUN mkdir -p /etc/tobiko RUN printf "[DEFAULT]\nlog_dir=${TOBIKO_REPORT_DIR}" > /etc/tobiko/tobiko.conf -# Copy python pacakges -COPY --from=install /usr/local /usr/local/ - # Copy tobiko tools ADD tools/ ${TOBIKO_DIR}/tools/ +# Copy python pacakges +COPY --from=install /usr/local /usr/local/ + WORKDIR ${TOBIKO_DIR} CMD tools/run_tests.py ${OS_TEST_PATH} + + +FROM tobiko as linters + +ENV SKIP=check-executables-have-shebangs,pylint + +# Copy configuration files +ADD .pre-commit-config.yaml \ + linters-requirements.txt \ + ${TOBIKO_DIR}/ + +RUN python3 -m pip install -r ${TOBIKO_DIR}/linters-requirements.txt +RUN pre-commit install --install-hooks + +WORKDIR ${TOBIKO_DIR} +CMD pre-commit run -a diff --git a/docker-compose.yml b/docker-compose.yml index c3fe98132..fb78c960b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,30 @@ services: - .:/tobiko - ~/.ssh:/root/.ssh + lower-constraints: + extends: + service: unit + build: + context: . + target: tobiko + args: + base_image: lower-constraints + + linters: + extends: + service: unit + build: + target: linters + + fedora: + extends: + service: unit + build: + context: . + target: tobiko + args: + base_image: fedora + functional: extends: service: unit diff --git a/tobiko/podman/_podman1/libs/tunnel.py b/tobiko/podman/_podman1/libs/tunnel.py index 48274ef17..331d862c0 100644 --- a/tobiko/podman/_podman1/libs/tunnel.py +++ b/tobiko/podman/_podman1/libs/tunnel.py @@ -81,8 +81,8 @@ class Portal(abc.MutableMapping): def _schedule_reaper(self): timer = threading.Timer(interval=self.sweap, function=self.reap) - timer.setName('PortalReaper') - timer.setDaemon(True) + timer.name = 'PortalReaper' + timer.daemon = True timer.start() def reap(self): diff --git a/tobiko/tests/functional/openstack/test_topology.py b/tobiko/tests/functional/openstack/test_topology.py index e7ec27df8..5c58a4fd9 100644 --- a/tobiko/tests/functional/openstack/test_topology.py +++ b/tobiko/tests/functional/openstack/test_topology.py @@ -15,7 +15,7 @@ # under the License. from __future__ import absolute_import -import collections +from collections import abc import random import re @@ -91,7 +91,7 @@ class OpenStackTopologyTest(testtools.TestCase): else: self.fail(f"Any node {node.name} group matches " f"'{group}': {node.groups}") - elif isinstance(group, collections.Iterable): + elif isinstance(group, abc.Iterable): matching_groups = set(group) & set(node.groups) self.assertNotEqual(set(), matching_groups, f"Any group of node {node.name} " diff --git a/zuul.d/docker.yaml b/zuul.d/docker.yaml index 13ee5b862..1f4d43ddd 100644 --- a/zuul.d/docker.yaml +++ b/zuul.d/docker.yaml @@ -6,9 +6,12 @@ Docker jobs intended to gate Tobiko container related changes check: &CHECK jobs: + - tobiko-docker + - tobiko-docker-fedora - tobiko-docker-functional - tobiko-docker-infrared - - tobiko-docker-unit + - tobiko-docker-linters + - tobiko-docker-lower-constraints gate: jobs: - tobiko-docker-functional @@ -18,7 +21,7 @@ # ---------------------------------------------------------------------------- - job: - name: tobiko-docker-unit + name: tobiko-docker description: | Run unit tests for an OpenStack Python project inside a Docker container. voting: true @@ -32,12 +35,23 @@ docker_compose_service: unit +- job: + name: tobiko-docker-fedora + description: | + Run functional tests for an OpenStack Python project inside a Docker container + based on Fedora 35 + voting: true + parent: tobiko-docker + vars: + docker_compose_service: fedora + + - job: name: tobiko-docker-functional description: | Run functional tests for an OpenStack Python project inside a Docker container. voting: true - parent: tobiko-docker-unit + parent: tobiko-docker timeout: 3600 vars: docker_compose_service: functional @@ -48,7 +62,28 @@ description: | Run functional tests for an OpenStack Python project inside a Docker container. voting: true - parent: tobiko-docker-unit + parent: tobiko-docker timeout: 3600 vars: docker_compose_service: infrared + + +- job: + name: tobiko-docker-linters + description: | + Run static analisys verifications + voting: true + parent: tobiko-docker + vars: + docker_compose_service: linters + + +- job: + name: tobiko-docker-lower-constraints + description: | + Run functional tests for an OpenStack Python project inside a Docker container + with lower-constrains.txt requirements file. + voting: true + parent: tobiko-docker + vars: + docker_compose_service: lower-constraints