diff --git a/Dockerfile b/Dockerfile index 4d841476..5ed014ff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,17 @@ +ARG WHEELS=quay.io/airshipit/requirements:master-ubuntu_jammy ARG FROM=ubuntu:jammy -FROM ${FROM} + +# This is an alias for mounting the wheels image +FROM ${WHEELS} AS wheels + +FROM ${FROM} AS common ENV PATH=/var/lib/openstack/bin:$PATH ENV LANG=C.UTF-8 + +# WHEELS_PATH must not be somewhere in /tmp because /tmp/* are deleted in the end of build +ARG WHEELS_PATH="/wheels" ARG PROJECT -ARG WHEELS=loci/requirements:master-ubuntu_jammy ARG PROJECT_REPO=https://opendev.org/openstack/${PROJECT} ARG PROJECT_REF=master ARG PROJECT_RELEASE=master @@ -56,4 +63,8 @@ ADD data /tmp/ COPY scripts /opt/loci/scripts ADD bindep.txt pydep.txt $EXTRA_BINDEP $EXTRA_PYDEP /opt/loci/ +FROM common AS requirements RUN /opt/loci/scripts/install.sh + +FROM common AS project +RUN --mount=type=bind,from=wheels,target=${WHEELS_PATH} /opt/loci/scripts/install.sh diff --git a/README.md b/README.md index 49430f7f..a7844e43 100644 --- a/README.md +++ b/README.md @@ -75,9 +75,9 @@ For more advanced building you can use docker build arguments to define: * `PROJECT_PIP_EXTRAS` python extras to use during project install. * `UID` The uid of the user that will be created (defaults to 42424). * `GID` The gid of the group that will be created (default to 42424). - * `WHEELS` The location of the wheels tarball. This accepts a url to a - tarball or a Docker image name in the form of - `[myregistry/]mydockernamespace/requirements[:ubuntu]` + * `WHEELS` The location of the wheels Docker image. The image must contain + wheels in the root directory. It is mounted while building other images. + `[myregistry/]mydockernamespace/requirements[:tag]` * `DISTRO` This is a helper variable used for scripts. It would primarily be used in situations where the script would not detect the correct distro. For example, you would set `DISTRO=centos` when running from an oraclelinux diff --git a/playbooks/build.yaml b/playbooks/build.yaml index bbe832f1..80d24e6a 100644 --- a/playbooks/build.yaml +++ b/playbooks/build.yaml @@ -51,16 +51,6 @@ loop_var: image_registry loop: "{{ image_registries }}" - - name: Tag requirements image and push to local registry - shell: | - for tag in {{ requirements_image.tags | join(" ") }}; do - docker tag {{ requirements_image.repository }}:$tag {{ local_registry }}/{{ requirements_image.repository }}:$tag - docker push {{ local_registry }}/{{ requirements_image.repository }}:$tag - done - loop_control: - loop_var: image_registry - loop: "{{ image_registries }}" - - name: Build project images include_role: name: build-container-image diff --git a/playbooks/pre-build.yaml b/playbooks/pre-build.yaml index 23a14f1f..0a19deb0 100644 --- a/playbooks/pre-build.yaml +++ b/playbooks/pre-build.yaml @@ -1,36 +1,7 @@ -- hosts: all[0] +- hosts: all gather_facts: true roles: - ensure-python - ensure-pip - clear-firewall - ensure-docker - - tasks: - - name: Vars - include_vars: - file: "vars.yaml" - - - name: Create docker directory - become: yes - file: - state: directory - path: /etc/docker - mode: 0755 - - - name: Configure docker daemon - become: yes - template: - dest: /etc/docker/daemon.json - group: root - mode: 0644 - owner: root - src: files/daemon.json.j2 - - - name: Print docker config - command: "cat /etc/docker/daemon.json" - - # This is necessary to serve requirements image to fetch wheels - # during building project images - - name: Start local registry - command: "docker run -d -p {{ local_registry }}:5000 --restart always --name registry {{ local_registry_image }}" diff --git a/playbooks/vars.yaml b/playbooks/vars.yaml index a07de2a4..2df25c61 100644 --- a/playbooks/vars.yaml +++ b/playbooks/vars.yaml @@ -23,6 +23,7 @@ requirements_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "requirements" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/requirements" tags: *tags @@ -36,6 +37,7 @@ barbican_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "project" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/barbican" tags: *tags @@ -44,7 +46,7 @@ barbican_image: - "PROJECT='barbican'" - "PROJECT_REF={{ openstack_release }}" - "PROFILES='fluent'" - - "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" + - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" - "KEEP_ALL_WHEELS=yes" - "DIST_PACKAGES='python3-dev gcc'" - "PIP_ARGS='--only-binary :none:'" @@ -53,6 +55,7 @@ cinder_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "project" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/cinder" tags: *tags @@ -62,13 +65,14 @@ cinder_image: - "PROJECT_REF={{ openstack_release }}" - "PROFILES='fluent lvm ceph qemu apache'" - "PIP_PACKAGES='python-swiftclient'" - - "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" + - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" - "KEEP_ALL_WHEELS=yes" cyborg_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "project" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/cyborg" tags: *tags @@ -77,13 +81,14 @@ cyborg_image: - "PROJECT='cyborg'" - "PROJECT_REF={{ openstack_release }}" - "DIST_PACKAGES='pciutils'" - - "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" + - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" - "KEEP_ALL_WHEELS=yes" designate_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "project" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/designate" tags: *tags @@ -92,13 +97,14 @@ designate_image: - "PROJECT='designate'" - "PROJECT_REF={{ openstack_release }}" - "PROFILES='fluent'" - - "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" + - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" - "KEEP_ALL_WHEELS=yes" glance_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "project" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/glance" tags: *tags @@ -108,13 +114,14 @@ glance_image: - "PROJECT_REF={{ openstack_release }}" - "PROFILES='fluent ceph qemu-utils'" - "PIP_PACKAGES='python-swiftclient os-brick python-cinderclient oslo-rootwrap'" - - "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" + - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" - "KEEP_ALL_WHEELS=yes" heat_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "project" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/heat" tags: *tags @@ -123,7 +130,7 @@ heat_image: - "PROJECT='heat'" - "PROJECT_REF={{ openstack_release }}" - "PROFILES='fluent apache'" - - "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" + - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" - "KEEP_ALL_WHEELS=yes" - "DIST_PACKAGES='curl'" @@ -131,6 +138,7 @@ horizon_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "project" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/horizon" tags: *tags @@ -139,7 +147,7 @@ horizon_image: - "PROJECT='horizon'" - "PROJECT_REF={{ openstack_release }}" - "PROFILES='fluent apache'" - - "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" + - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" - "KEEP_ALL_WHEELS=yes" - "PIP_PACKAGES='pymemcache'" @@ -147,6 +155,7 @@ ironic_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "project" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/ironic" tags: *tags @@ -156,13 +165,14 @@ ironic_image: - "PROJECT_REF={{ openstack_release }}" - "PROFILES='fluent ipxe ipmi qemu tftp'" - "DIST_PACKAGES='ethtool lshw iproute2'" - - "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" + - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" - "KEEP_ALL_WHEELS=yes" keystone_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "project" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/keystone" tags: *tags @@ -172,13 +182,14 @@ keystone_image: - "PROJECT_REF={{ openstack_release }}" - "PROFILES='fluent apache ldap {{ openstack_release }}'" - "PIP_PACKAGES='python-openstackclient'" - - "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" + - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" - "KEEP_ALL_WHEELS=yes" manila_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "project" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/manila" tags: *tags @@ -186,13 +197,14 @@ manila_image: - "FROM='base:{{ image_tag }}'" - "PROJECT='manila'" - "PROJECT_REF={{ openstack_release }}" - - "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" + - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" - "KEEP_ALL_WHEELS=yes" monasca_api_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "project" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/monasca-api" tags: *tags @@ -202,13 +214,14 @@ monasca_api_image: - "PROJECT_REF={{ openstack_release }}" - "PROFILES='monasca api'" - "PIP_PACKAGES='influxdb cassandra-driver sqlalchemy'" - - "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" + - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" - "KEEP_ALL_WHEELS=yes" neutron_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "project" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/neutron" tags: *tags @@ -219,13 +232,14 @@ neutron_image: - "PROFILES='fluent linuxbridge openvswitch apache vpn baremetal'" - "PIP_PACKAGES='tap-as-a-service'" - "DIST_PACKAGES='jq ethtool lshw'" - - "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" + - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" - "KEEP_ALL_WHEELS=yes" nova_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "project" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/nova" tags: *tags @@ -235,13 +249,14 @@ nova_image: - "PROJECT_REF={{ openstack_release }}" - "PROFILES='fluent ceph linuxbridge openvswitch configdrive qemu apache migration'" - "DIST_PACKAGES='net-tools openssh-server'" - - "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" + - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" - "KEEP_ALL_WHEELS=yes" octavia_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "project" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/octavia" tags: *tags @@ -249,7 +264,7 @@ octavia_image: - "FROM='base:{{ image_tag }}'" - "PROJECT='octavia'" - "PROJECT_REF={{ openstack_release }}" - - "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" + - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" - "KEEP_ALL_WHEELS=yes" openstack_client_pip_packages: @@ -280,6 +295,7 @@ openstack_client_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "project" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/openstack-client" tags: *tags @@ -287,7 +303,7 @@ openstack_client_image: - "FROM='base:{{ image_tag }}'" - "PROJECT='python-openstackclient'" - "PROJECT_REF={{ openstack_release }}" - - "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" + - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" - "KEEP_ALL_WHEELS=yes" - "PIP_PACKAGES='{{ openstack_client_pip_packages | join(' ') }}'" @@ -295,6 +311,7 @@ placement_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "project" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/placement" tags: *tags @@ -304,13 +321,14 @@ placement_image: - "PROJECT_REF={{ openstack_release }}" - "PROFILES='apache'" - "PIP_PACKAGES='httplib2'" - - "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" + - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" - "KEEP_ALL_WHEELS=yes" tacker_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "project" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/tacker" tags: *tags @@ -318,13 +336,14 @@ tacker_image: - "FROM='base:{{ image_tag }}'" - "PROJECT='tacker'" - "PROJECT_REF={{ openstack_release }}" - - "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" + - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" - "KEEP_ALL_WHEELS=yes" watcher_image: context: "." dockerfile: "Dockerfile" container_filename: "Dockerfile" + target: "project" registry: "{{ image_registry.host }}" repository: "{{ image_registry.host }}/{{ image_registry.org }}/watcher" tags: *tags @@ -332,5 +351,5 @@ watcher_image: - "FROM='base:{{ image_tag }}'" - "PROJECT='watcher'" - "PROJECT_REF={{ openstack_release }}" - - "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" + - "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'" - "KEEP_ALL_WHEELS=yes" diff --git a/scripts/fetch_wheels.py b/scripts/fetch_wheels.py deleted file mode 100755 index 7f28e118..00000000 --- a/scripts/fetch_wheels.py +++ /dev/null @@ -1,210 +0,0 @@ -#!/usr/bin/env python -import json -import os -import platform -import re -import ssl -from urllib import request as urllib2 -import logging -import sys - -DOCKER_REGISTRY = 'registry.hub.docker.com' - -MANIFEST_V1 = 'application/vnd.oci.image.manifest.v1+json' -MANIFEST_V2 = 'application/vnd.docker.distribution.manifest.v2+json' -MANIFEST_V2_LIST = 'application/vnd.docker.distribution.manifest.list.v2+json' - -ARCH_MAP = { - 'x86_64': 'amd64', - 'aarch64': 'arm64', -} - - -logging.basicConfig(level=logging.INFO, stream=sys.stdout) -log = logging.getLogger(__name__) - - -def strtobool(v): - # Clone from the now-deprecated distutils - return str(v).lower() in ("yes", "true", "t", "1") - - -def registry_urlopen(r): - if strtobool(os.environ.get('REGISTRY_INSECURE', "False")): - resp = urllib2.urlopen(r, context=ssl._create_unverified_context()) - else: - resp = urllib2.urlopen(r) - return resp - - -def registry_request(r, token=None): - try: - if token: - r.add_header('Authorization', 'Bearer {}'.format(token)) - return registry_urlopen(r) - except urllib2.HTTPError as err: - if err.reason == 'Unauthorized' and token is None: - value = err.headers['www-authenticate'].split(' ', 2) - items = urllib2.parse_http_list(value[1]) - opts = urllib2.parse_keqv_list(items) - - url = "{}?service={}&scope={}".format( - opts['realm'], - opts['service'], - opts['scope'] - ) - - auth_request = urllib2.Request(url=url) - resp = registry_urlopen(auth_request) - resp_text = resp.read().decode('utf-8').strip() - token = json.loads(resp_text)['token'] - - return registry_request(r, token) - raise - - -def get_sha(repo, tag, registry, protocol): - headers = { - 'Accept': ', '.join([MANIFEST_V2_LIST, MANIFEST_V2, MANIFEST_V1]) - } - url = "{}://{}/v2/{}/manifests/{}".format(protocol, registry, repo, tag) - print(url) - r = urllib2.Request(url=url, headers=headers) - resp = registry_request(r) - resp_text = resp.read().decode('utf-8').strip() - manifest = json.loads(resp_text) - if manifest['schemaVersion'] == 1: - return manifest['fsLayers'][0]['blobSum'] - elif manifest['schemaVersion'] == 2: - if manifest['mediaType'] == MANIFEST_V2_LIST: - arch = platform.processor() - - if arch not in ARCH_MAP: - raise SystemError("Unknown architecture: %s" % arch) - - for m in manifest['manifests']: - # NOTE(mnaser): At this point, we've found the digest for the - # manifest we want, we go back and run this code - # again but getting that arch-specific manifest. - if m['platform']['architecture'] == ARCH_MAP[arch]: - tag = m['digest'] - return get_sha(repo, tag, registry, protocol) - - # NOTE(mnaser): If we're here, we've gone over all the manifests - # and we didn't find one that matches our requested - # architecture. - raise SystemError("Manifest does not include arch: %s" % - ARCH_MAP[arch]) - else: - # NOTE(mnaser): This is the cause if the registry returns a manifest - # which isn't a list (single arch cases or getting - # a specific arch from a manifest list). The V2 - # manifest orders layers from base to end (as opposed - # to V1) so we need to get the last digest. - return manifest['layers'][-1]['digest'] - raise SystemError("Unable to find correct manifest schema version") - - -def get_blob(repo, tag, protocol, registry=DOCKER_REGISTRY): - log.info("Fetching blob from %s://%s/%s:%s", protocol, registry, repo, tag) - sha = get_sha(repo, tag, registry, protocol) - url = "{}://{}/v2/{}/blobs/{} ".format(protocol, registry, repo, sha) - print(url) - r = urllib2.Request(url=url) - resp = registry_request(r) - return resp.read() - - -def protocol_detection(registry, protocol='http'): - PROTOCOLS = ('http', 'https') - index = PROTOCOLS.index(protocol) - try: - url = "{}://{}".format(protocol, registry) - log.info("Trying to connect to: %s", url) - r = urllib2.Request(url) - urllib2.urlopen(r) - except (urllib2.URLError, urllib2.HTTPError) as err: - log.info("Failed to connect to %s://%s", protocol, registry) - if err.reason == 'Forbidden': - log.info("Forbidden. Protocol detected: %s", protocol) - return protocol - elif index < len(PROTOCOLS) - 1: - return protocol_detection(registry, PROTOCOLS[index + 1]) - else: - raise Exception("Cannot detect protocol for registry: {} due to error: {}".format(registry, err)) - else: - log.info("Protocol detected: %s", protocol) - return protocol - - -def get_wheels(url): - r = urllib2.Request(url=url) - resp = registry_request(r) - # Using urllib2.request.urlopen() from python3 will face the IncompleteRead and then system report connect refused. - # To avoid this problem, add an exception to ensure that all packages will be transmitted. before link down. - try: - buf = resp.read() - except Exception as e: - buf = e.partial - return buf - - -def parse_image(full_image): - slash_occurrences = len(re.findall('/', full_image)) - repo = None - registry = DOCKER_REGISTRY - if slash_occurrences > 1: - full_image_list = full_image.split('/') - registry = full_image_list[0] - repo = '/'.join(full_image_list[1:-1]) - image = full_image_list[-1] - elif slash_occurrences == 1: - repo, image = full_image.split('/') - else: - image = full_image - if image.find(':') != -1: - image, tag = image.split(':') - else: - tag = 'latest' - return registry, repo + '/' + image if repo else image, tag - - -def main(): - if 'WHEELS' in os.environ: - wheels = os.environ['WHEELS'] - else: - with open('/opt/loci/wheels', 'r') as f: - wheels = f.read() - - if wheels.startswith('/'): - with open(wheels, 'rb') as f: - data = f.read() - elif wheels.startswith('http'): - data = get_wheels(wheels) - else: - registry, image, tag = parse_image(wheels) - log.info("Image parsed: wheels=%s registry=%s image=%s tag=%s", - wheels, registry, image, tag) - if os.environ.get('REGISTRY_PROTOCOL') in ['http', 'https']: - protocol = os.environ.get('REGISTRY_PROTOCOL') - elif os.environ.get('REGISTRY_PROTOCOL') == 'detect': - protocol = protocol_detection(registry) - else: - raise ValueError("Unknown protocol given in argument") - kwargs = dict() - if registry: - kwargs.update({'registry': registry}) - data = get_blob(image, tag, protocol, **kwargs) - - if 'WHEELS_DEST' in os.environ: - dest = os.environ['WHEELS_DEST'] - else: - with open('/opt/loci/wheels', 'w') as f: - f.write(wheels) - dest = '/tmp/wheels.tar.gz' - with open(dest, 'wb') as f: - f.write(data) - - -if __name__ == '__main__': - main() diff --git a/scripts/fetch_wheels.sh b/scripts/fetch_wheels.sh deleted file mode 100755 index de4548b2..00000000 --- a/scripts/fetch_wheels.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -ex - -python3 $(dirname $0)/fetch_wheels.py - -mkdir -p /tmp/wheels/ -# Exclude all files starting with '.' as these can be control files for -# AUFS which have special meaning on AUFS backed file stores. -tar xf /tmp/wheels.tar.gz --exclude='.*' -C /tmp/wheels/ - -# Presence of constraint for project we build (in Stein, Train for Horizon and -# Neutron) in uc breaks project installation with unsatisfied constraints error -# This line ensures that such constraint is absent for any future occurrence -sed -i "/^${PROJECT}===/d" /tmp/wheels/upper-constraints.txt diff --git a/scripts/install.sh b/scripts/install.sh index 447cfc61..c95deaa7 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -63,7 +63,6 @@ if [ "${KEEP_ALL_WHEELS}" != "False" ]; then NO_INDEX=--no-index fi -$(dirname $0)/fetch_wheels.sh if [[ "${PROJECT}" == "infra" ]]; then $(dirname $0)/setup_pip.sh $(dirname $0)/pip_install.sh bindep ${PIP_PACKAGES} diff --git a/scripts/pip_install.sh b/scripts/pip_install.sh index c7297de3..cde468f4 100755 --- a/scripts/pip_install.sh +++ b/scripts/pip_install.sh @@ -4,4 +4,11 @@ set -ex packages=$@ -pip install --no-cache-dir --only-binary :all: --no-compile -c /tmp/wheels/global-requirements.txt -c /tmp/wheels/upper-constraints.txt --find-links /tmp/wheels/ --ignore-installed ${PIP_ARGS} ${packages} +# Presence of constraint for project we build +# in upper constraints breaks project installation +# with unsatisfied constraints error. +# This line ensures that such constraint is absent. +cp ${WHEELS_PATH}/upper-constraints.txt /tmp/upper-constraints.txt +sed -i "/^${PROJECT}===/d" /tmp/upper-constraints.txt + +pip install --no-cache-dir --only-binary :all: --no-compile -c ${WHEELS_PATH}/global-requirements.txt -c /tmp/upper-constraints.txt --find-links ${WHEELS_PATH} --ignore-installed ${PIP_ARGS} ${packages}