[WIP] Add Ubuntu Focal image and py38 support

- update "determine tags" task in docker-image-build to use `python3` instead of `python` - update zuul jobs to use focal - update zuul nodesets to focal - update zuul to run py38 instead of py36 jobs - use new pass_env syntax in tox.ini - replace whitelist_externals with allowlist_externals in tox.ini - apparmor utils now required when building docker images - update Makefile to use focal by default - update python dependencies - replace deprecated `setup()` with `setup_method()` in tests [1] - setup_method `method` param not optional anymore when decorated - silence bandit false positive in passphrase_generator.py - update test_generate_keypair to check for either PKCS#1 or PKCS#8 formatted keys. Ubuntu Jammy updated to openssl 3.x which changes the default format [2] [1] https://docs.pytest.org/en/stable/deprecations.html#support-for-tests-written-for-nose [2] https://www.openssl.org/news/cl30.txt Change-Id: I46e579f2d8d3f50f49faa22457b3cde685f4e789
2023-02-07 15:31:04 -07:00 · 2023-02-07 15:31:04 -07:00 · e49cfcba44
commit e49cfcba44
parent 770cf4256e
11 changed files with 1299 additions and 831 deletions
--- a/.zuul.yaml
+++ b/.zuul.yaml
@ -13,7 +13,7 @@
 - project:
    templates:
      - docs-on-readthedocs
-      - openstack-python36-jobs
+      - openstack-python38-jobs
      - openstack-cover-jobs
      - release-notes-jobs-python3
    vars:
@ -29,87 +29,88 @@
      base_image_bionic: "ubuntu:18.04"
      base_image_opensuse: "opensuse/leap:15.1"
      base_image_xenial: "ubuntu:16.04"
+      base_image_focal: "ubuntu:20.04"
    check:
      jobs:
-        - openstack-tox-pep8-bionic
+        - openstack-tox-pep8-focal
        - pegleg-dependency-vulnerability-check
-        - airship-pegleg-docker-build-gate-ubuntu_bionic
+        - airship-pegleg-docker-build-gate-ubuntu_focal
        - airship-pegleg-lint-yaml

    gate:
      jobs:
-        - openstack-tox-pep8-bionic
+        - openstack-tox-pep8-focal
        - pegleg-dependency-vulnerability-check
-        - airship-pegleg-docker-build-gate-ubuntu_bionic
+        - airship-pegleg-docker-build-gate-ubuntu_focal
        - airship-pegleg-lint-yaml
    post:
      jobs:
-        - airship-pegleg-docker-publish-ubuntu_bionic
+        - airship-pegleg-docker-publish-ubuntu_focal
        - pegleg-upload-git-mirror

- nodeset:
-    name: airship-pegleg-single-node
-    nodes:
-      - name: primary
-        label: ubuntu-xenial
-
 - nodeset:
    name: airship-pegleg-single-node-bionic
    nodes:
      - name: primary
        label: ubuntu-bionic

+- nodeset:
+    name: airship-pegleg-single-node-focal
+    nodes:
+      - name: primary
+        label: ubuntu-focal
+
 - job:
-    name: openstack-tox-pep8-bionic
+    name: openstack-tox-pep8-focal
    parent: openstack-tox-pep8
-    nodeset: openstack-single-node-bionic
+    nodeset: openstack-single-node-focal

 - job:
    name: airship-pegleg-lint-yaml
    voting: true
    timeout: 600
    run: tools/gate/playbooks/lint-yaml.yaml
-    nodeset: ubuntu-bionic
+    nodeset: ubuntu-focal
    irrelevant-files:
      - ^.*\.rst$
      - ^doc/.*$

 - job:
    name: pegleg-dependency-vulnerability-check
-    parent: tox-py36
+    parent: tox-py38
    voting: false
    timeout: 600
-    nodeset: ubuntu-bionic
+    nodeset: ubuntu-focal
    vars:
      tox_envlist: safety
-      bindep_profile: test py36
+      bindep_profile: test py38

 - job:
-    name: airship-pegleg-docker-build-gate-ubuntu_bionic
+    name: airship-pegleg-docker-build-gate-ubuntu_focal
    timeout: 1800
    run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: airship-pegleg-single-node-bionic
+    nodeset: airship-pegleg-single-node-focal
    irrelevant-files:
      - '^doc/.*'
    vars:
      publish: false
-      distro: ubuntu_bionic
+      distro: ubuntu_focal
      tags:
        dynamic:
          patch_set: true

 - job:
-    name: airship-pegleg-docker-publish-ubuntu_bionic
+    name: airship-pegleg-docker-publish-ubuntu_focal
    timeout: 1800
    run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: airship-pegleg-single-node-bionic
+    nodeset: airship-pegleg-single-node-focal
    secrets:
      - airship_pegleg_quay_creds
    irrelevant-files:
      - '^doc/.*'
    vars:
      publish: true
-      distro: ubuntu_bionic
+      distro: ubuntu_focal
      tags:
        dynamic:
          branch: true
--- a/4
+++ b/4
@ -27,9 +27,9 @@ PUSH_IMAGE        ?= false
 # use this variable for image labels added in internal build process
 LABEL             ?= org.airshipit.build=community
 COMMIT            ?= $(shell git rev-parse HEAD)
-DISTRO            ?= ubuntu_bionic
+DISTRO            ?= ubuntu_focal
 IMAGE             ?= $(DOCKER_REGISTRY)/$(IMAGE_PREFIX)/$(IMAGE_NAME):$(IMAGE_TAG)-${DISTRO}
-PYTHON_BASE_IMAGE ?= python:3.6
+PYTHON_BASE_IMAGE ?= python:3.8
 BASE_IMAGE        ?=

 export
--- a/18
+++ b/18
@ -4,8 +4,8 @@ url = "https://pypi.org/simple"
 verify_ssl = true

 [dev-packages]
-pytest = "~=6.1.1"
-pytest-cov = "~=2.10.1"
+pytest = "~=7.2.1"
+pytest-cov = "~=4.0"
 testfixtures = "*"
 pytest-xdist = "*"
 requests = ">=2.20.0"
@ -24,15 +24,17 @@ python-dateutil = ">=2.8.1"
 GitPython = "~=2.1.11"
 docker = ">=3.7.2"
 pylibyaml = "~=0.1"
-PyYAML = "~=5.1"
-deckhand = {git = "https://opendev.org/airship/deckhand.git",ref = "5cd799cc5d04527ac782270008ff647b3779ff05"}
-shipyard-client = {git = "https://opendev.org/airship/shipyard.git",ref = "e3e71f7d1632c0dc13b436a32c238f9da76d6d88",subdirectory = "src/bin/shipyard_client"}
-promenade = {git = "https://opendev.org/airship/promenade.git",ref = "c10165c144e0a18137596a3c89f1339d6ed30d0c"}
-"oslo.i18n" = "~=3.24.0"
+PyYAML = "~=5.4.1"
+deckhand = {git = "https://opendev.org/airship/deckhand.git",ref = "70aa35a396d5f76753616f5289228f9c2b0e7ec7"}
+#shipyard-client = {git = "https://opendev.org/airship/shipyard.git",ref = "e3e71f7d1632c0dc13b436a32c238f9da76d6d88",subdirectory = "src/bin/shipyard_client"}
+shipyard-client = {git = "https://opendev.org/airship/shipyard.git",ref = "refs/changes/15/874815/1",subdirectory = "src/bin/shipyard_client"}
+promenade = {git = "https://opendev.org/airship/promenade.git",ref = "8ce937a9f7ac94d51d4d946b6c2288975c53853a"}
+"oslo.i18n" = ">=3.24.0"
 "oslo.utils" = ">=3.42.1"
 six = ">=1.15.0"
 debtcollector = "~=1.22.0"
 zipp = "~=3.6.0"
+psycopg2-binary = "*"

 [requires]
-python_version = "3.6"
+python_version = "3.8"
--- a/Pipfile.lock
+++ b/Pipfile.lock
--- a/images/pegleg/Dockerfile.ubuntu_focal
+++ b/images/pegleg/Dockerfile.ubuntu_focal
@ -0,0 +1,75 @@
+ARG FROM=ubuntu:20.04
+FROM ${FROM}
+ARG CFSSLURL=https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
+
+LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode'
+LABEL org.opencontainers.image.url='https://airshipit.org'
+LABEL org.opencontainers.image.documentation='https://airship-pegleg.readthedocs.org'
+LABEL org.opencontainers.image.source='https://opendev.org/airship/pegleg'
+LABEL org.opencontainers.image.vendor='The Airship Authors'
+LABEL org.opencontainers.image.licenses='Apache-2.0'
+
+ENV LANG=C.UTF-8
+ENV LC_ALL=C.UTF-8
+ARG DECKHAND_VERSION=branch/master
+ARG SHIPYARD_VERSION=branch/master
+ARG PROMENADE_VERSION=branch/master
+
+RUN set -ex \
+    && apt-get update -qq \
+    && apt-get install -y --no-install-recommends \
+         automake \
+         ca-certificates \
+         curl \
+         gcc \
+         git \
+         gpg \
+         gpg-agent \
+         libssl-dev \
+         libtool \
+         make \
+         netbase \
+         openssh-client \
+         python3-dev \
+         python3-pip \
+         python3-setuptools \
+    && python3 -m pip install -U pip \
+    && apt-get autoremove -yqq --purge \
+    && apt-get clean \
+    && rm -rf \
+         /tmp/* \
+         /usr/share/doc \
+         /usr/share/doc-base \
+         /usr/share/man \
+         /var/lib/apt/lists/* \
+         /var/log/* \
+         /var/tmp/*
+
+ENV LD_LIBRARY_PATH=/usr/local/lib
+
+ARG LIBYAML_VERSION=0.2.5
+RUN set -ex \
+    && git clone https://github.com/yaml/libyaml.git \
+    && cd libyaml \
+    && git checkout $LIBYAML_VERSION \
+    && ./bootstrap \
+    && ./configure \
+    && make \
+    && make install \
+    && cd .. \
+    && rm -fr libyaml
+
+VOLUME /var/pegleg
+WORKDIR /var/pegleg
+
+COPY requirements.txt /opt/pegleg/requirements.txt
+RUN pip3 install -r https://opendev.org/airship/deckhand/raw/${DECKHAND_VERSION}/requirements.txt \
+    && pip3 install -r https://opendev.org/airship/promenade/raw/${PROMENADE_VERSION}/requirements.txt \
+    && pip3 install -r https://opendev.org/airship/shipyard/raw/${SHIPYARD_VERSION}/src/bin/shipyard_client/requirements.txt \
+    && pip3 install --no-cache-dir -r /opt/pegleg/requirements.txt
+
+COPY tools/install-cfssl.sh /opt/pegleg/tools/install-cfssl.sh
+RUN /opt/pegleg/tools/install-cfssl.sh ${CFSSLURL}
+
+COPY . /opt/pegleg
+RUN pip3 install -e /opt/pegleg
--- a/pegleg/engine/generators/passphrase_generator.py
+++ b/pegleg/engine/generators/passphrase_generator.py
@ -102,7 +102,7 @@ class PassphraseGenerator(BaseGenerator):
                        self.validate_base64,
                        auto_allowed=auto_allowed)

-                elif passphrase_type == 'passphrase':
+                elif passphrase_type == 'passphrase':  # nosec
                    passphrase = self._prompt_user_passphrase_and_validate(
                        p_name,
                        'passphrase',
--- a/tests/unit/cli/test_commands.py
+++ b/tests/unit/cli/test_commands.py
@ -30,7 +30,7 @@ from tests.unit import test_utils
 TEST_PARAMS = {
    "site_name": "seaworthy",
    "site_type": "foundry",
-    "repo_rev": '342c9eedd58f046a322ee2dd6752a9ec8fa992bb',
+    "repo_rev": '23e4bfc80aa6a3c845b31750d7593898f6e55e82',
    "repo_name": "treasuremap",
    "repo_url": "https://review.opendev.org/airship/treasuremap.git",
 }
@ -756,7 +756,7 @@ class TestSiteSecretsActions(BaseCLIActionTest):

 class TestTypeCliActions(BaseCLIActionTest):
    """Tests type-level CLI actions."""
-    def setup(self):
+    def setup_method(self):
        self.expected_types = ['foundry']

    def _assert_table_has_expected_sites(self, table_output):
@ -795,7 +795,7 @@ class TestTypeCliActions(BaseCLIActionTest):

 class TestSiteCliActionsWithSubdirectory(BaseCLIActionTest):
    """Tests site CLI actions with subdirectories in repository paths."""
-    def setup(self):
+    def setup_method(self):
        self.expected_sites = ['demo', 'gate-multinode', 'dev', 'dev-proxy']

    def _assert_table_has_expected_sites(self, table_output):
@ -874,7 +874,7 @@ class TestCliSiteSubcommandsWithDecryptOption(BaseCLIActionTest):
            "PEGLEG_PASSPHRASE": 'ytrr89erARAiPE34692iwUMvWqqBvC',
            "PEGLEG_SALT": "MySecretSalt1234567890]["
        })
-    def setup(self):
+    def setup_method(self, method):
        pegleg_main.run_config(
            self.treasuremap_path, None, None, None, [], True, False)
        pegleg_main.run_encrypt('zuul-tester', None, self.site_name)
--- a/tests/unit/engine/catalog/test_pki_utility.py
+++ b/tests/unit/engine/catalog/test_pki_utility.py
@ -25,7 +25,8 @@ from pegleg.engine.common import managed_document
 CERT_HEADER = '-----BEGIN CERTIFICATE-----\n'
 CERT_KEY_HEADER = '-----BEGIN RSA PRIVATE KEY-----\n'
 PUBLIC_KEY_HEADER = '-----BEGIN PUBLIC KEY-----\n'
-PRIVATE_KEY_HEADER = '-----BEGIN RSA PRIVATE KEY-----\n'
+PRIVATE_KEY_HEADER_PKCS1 = '-----BEGIN RSA PRIVATE KEY-----\n'
+PRIVATE_KEY_HEADER_PKCS8 = '-----BEGIN PRIVATE KEY-----\n'

 PEGLEG_MANAGED_DOC_SCHEMA = 'pegleg/PeglegManagedDocument/v1'
 CA_SCHEMA = 'deckhand/CertificateAuthority/v1'
@ -87,7 +88,8 @@ class TestPKIUtility(object):

        assert isinstance(priv_key, dict), priv_key
        assert PRIVATE_KEY_SCHEMA in priv_key['schema']
-        assert PRIVATE_KEY_HEADER in priv_key['data']
+        assert PRIVATE_KEY_HEADER_PKCS1 in priv_key['data'] or \
+            PRIVATE_KEY_HEADER_PKCS8 in priv_key['data']

    def test_generate_certificate(self):
        pki_obj = pki_utility.PKIUtility(duration=365)
--- a/tests/unit/engine/test_selectable_linting.py
+++ b/tests/unit/engine/test_selectable_linting.py
@ -32,7 +32,7 @@ For more information, see: https://storyboard.openstack.org/#!/story/2003762


 class TestSelectableLinting(object):
-    def setup(self):
+    def setup_method(self):
        self.site_yaml_path = os.path.join(os.getcwd(), 'site_yamls')

    def _exclude_all(self, except_code):
--- a/tools/gate/playbooks/docker-image-build.yaml
+++ b/tools/gate/playbooks/docker-image-build.yaml
@ -29,23 +29,23 @@
        - debug:
            msg: "{{ tags | to_json }}"

+    - name: Set ubuntu focal base image
+      set_fact:
+        base_image: "{{ base_image_focal }}"
+      when: distro == 'ubuntu_focal'
+
    - name: Set ubuntu bionic base image
      set_fact:
        base_image: "{{ base_image_bionic }}"
      when: distro == 'ubuntu_bionic'

-    - name: Set ubuntu xenial base image
-      set_fact:
-        base_image: "{{ base_image_xenial }}"
-      when: distro == 'ubuntu_xenial'
-
    - name: Set ubuntu opensuse base image
      set_fact:
        base_image: "{{ base_image_opensuse }}"
      when: distro == 'opensuse_15'

    - name: Determine tags
-      shell: echo '{{ tags | to_json }}' | python {{ zuul.project.src_dir }}/tools/image_tags.py
+      shell: echo '{{ tags | to_json }}' | python3 {{ zuul.project.src_dir }}/tools/image_tags.py
      environment:
        BRANCH: "{{ zuul.branch | default('') }}"
        CHANGE: "{{ zuul.change | default('') }}"
@ -62,7 +62,7 @@
      become: True

    - name: Install Docker (Debian)
-      when: ansible_os_family == 'Debian'
+      when: ansible_facts['os_family'] == 'Debian'
      block:
        - file:
            path: "{{ item }}"
@ -86,11 +86,12 @@
        - apt_key:
            url: https://download.docker.com/linux/ubuntu/gpg
        - apt_repository:
-            repo: "deb http://{{ deb_docker_repo }} bionic stable"
+            repo: "deb http://{{ deb_docker_repo }} focal stable"
        - apt:
            name: "{{ item }}"
            allow_unauthenticated: True
          with_items:
+            - apparmor
            - docker-ce
            - python3-pip
            - python3-setuptools
--- a/tox.ini
+++ b/tox.ini
@ -1,6 +1,6 @@
 [tox]
-envlist = py36,pep8,docs,cover
-minversion = 2.3.1
+envlist = py38,pep8,docs,cover
+minversion = 3.4
 skipsdist = True

 [testenv]
@ -14,15 +14,22 @@ setenv =
 deps =
  pipenv
  setuptools < 58.0.0
-passenv = http_proxy https_proxy HTTP_PROXY HTTPS_PROXY no_proxy NO_PROXY PBR_VERSION
-whitelist_externals =
+passenv =
+  http_proxy
+  https_proxy
+  HTTP_PROXY
+  HTTPS_PROXY
+  no_proxy
+  NO_PROXY
+  PBR_VERSION
+allowlist_externals =
  bash
  find
 commands =
  pipenv install --dev
  find . -type f -name "*.pyc" -delete
  bash -c "{toxinidir}/tools/install-cfssl.sh"
-  {toxinidir}/tools/gate/run-unit-tests.sh '{posargs}'
+  bash -c "{toxinidir}/tools/gate/run-unit-tests.sh '{posargs}'"

 [testenv:fmt]
 basepython = python3
@ -46,7 +53,7 @@ commands =
  bandit -r pegleg -n 5
  flake8 {toxinidir}/pegleg
  yapf -dr {toxinidir}/pegleg {toxinidir}/tests
-whitelist_externals =
+allowlist_externals =
  bash

 [testenv:docs]
@ -58,7 +65,7 @@ deps =
 commands =
  pipenv install --dev
  bash -c "{toxinidir}/tools/gate/build-docs.sh"
-whitelist_externals =
+allowlist_externals =
  bash

 [testenv:bandit]
@ -84,11 +91,11 @@ deps =
  setuptools < 58.0.0
 commands =
  pipenv install --dev
-  {toxinidir}/tools/install-cfssl.sh
+  bash -c "{toxinidir}/tools/install-cfssl.sh"
  bash -c 'PATH=$PATH:~/.local/bin; pytest --cov=pegleg --cov-report \
      html:cover --cov-report xml:cover/coverage.xml --cov-report term \
      --cov-fail-under 87 tests/'
-whitelist_externals =
+allowlist_externals =
  bash

 [testenv:update-requirements]
@ -98,7 +105,7 @@ deps =
 commands =
  pipenv lock --clear
  bash -c "pipenv lock -r > {toxinidir}/requirements.txt"
-whitelist_externals =
+allowlist_externals =
  bash

 [testenv:releasenotes]
@ -107,7 +114,7 @@ deps = -r{toxinidir}/doc/requirements.txt
 commands =
  rm -rf releasenotes/build
  sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
-whitelist_externals =
+allowlist_externals =
  rm

 [testenv:venv]