openstack/loci
Code Issues Proposed changes

Mount requirements image

Browse Source 
Docker build allows you to mount images during
build to get access to their contents.

Let's use this functionality and mount requirements
image instead of fetching wheels directly from a registry
or via http.

Depends-On: Id2b78dba5c99f38599345f16707683f3bde533e4
Change-Id: Ia9eaf8598d2ef05f2bdf54f51c42d1ec4e4383df
This commit is contained in:
Vladimir Kozhukalov
2025-05-14 21:09:44 -05:00
parent 26c6e749eb
commit dd30388990
9 changed files with 62 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
Dockerfile
View File

@@ -1,10 +1,17 @@
ARG WHEELS=quay.io/airshipit/requirements:master-ubuntu_jammy
ARG FROM=ubuntu:jammy
FROM ${FROM}
# This is an alias for mounting the wheels image
FROM ${WHEELS} AS wheels
FROM ${FROM} AS common
ENV PATH=/var/lib/openstack/bin:$PATH
ENV LANG=C.UTF-8
# WHEELS_PATH must not be somewhere in /tmp because /tmp/* are deleted in the end of build
ARG WHEELS_PATH="/wheels"
ARG PROJECT
ARG WHEELS=loci/requirements:master-ubuntu_jammy
ARG PROJECT_REPO=https://opendev.org/openstack/${PROJECT}
ARG PROJECT_REF=master
ARG PROJECT_RELEASE=master
@@ -56,4 +63,8 @@ ADD data /tmp/
COPY scripts /opt/loci/scripts
ADD bindep.txt pydep.txt $EXTRA_BINDEP $EXTRA_PYDEP /opt/loci/
FROM common AS requirements
RUN /opt/loci/scripts/install.sh
FROM common AS project
RUN --mount=type=bind,from=wheels,target=${WHEELS_PATH} /opt/loci/scripts/install.sh

6
README.md
View File

@@ -75,9 +75,9 @@ For more advanced building you can use docker build arguments to define:
* `PROJECT_PIP_EXTRAS` python extras to use during project install.
* `UID` The uid of the user that will be created (defaults to 42424).
* `GID` The gid of the group that will be created (default to 42424).
* `WHEELS` The location of the wheels tarball. This accepts a url to a
tarball or a Docker image name in the form of
`[myregistry/]mydockernamespace/requirements[:ubuntu]`
* `WHEELS` The location of the wheels Docker image. The image must contain
wheels in the root directory. It is mounted while building other images.
`[myregistry/]mydockernamespace/requirements[:tag]`
* `DISTRO` This is a helper variable used for scripts. It would primarily be
used in situations where the script would not detect the correct distro.
For example, you would set `DISTRO=centos` when running from an oraclelinux

10
playbooks/build.yaml
View File

@@ -51,16 +51,6 @@
loop_var: image_registry
loop: "{{ image_registries }}"
- name: Tag requirements image and push to local registry
shell: |
for tag in {{ requirements_image.tags | join(" ") }}; do
docker tag {{ requirements_image.repository }}:$tag {{ local_registry }}/{{ requirements_image.repository }}:$tag
docker push {{ local_registry }}/{{ requirements_image.repository }}:$tag
done
loop_control:
loop_var: image_registry
loop: "{{ image_registries }}"
- name: Build project images
include_role:
name: build-container-image

31
playbooks/pre-build.yaml
View File

@@ -1,36 +1,7 @@
- hosts: all[0]
- hosts: all
gather_facts: true
roles:
- ensure-python
- ensure-pip
- clear-firewall
- ensure-docker
tasks:
- name: Vars
include_vars:
file: "vars.yaml"
- name: Create docker directory
become: yes
file:
state: directory
path: /etc/docker
mode: 0755
- name: Configure docker daemon
become: yes
template:
dest: /etc/docker/daemon.json
group: root
mode: 0644
owner: root
src: files/daemon.json.j2
- name: Print docker config
command: "cat /etc/docker/daemon.json"
# This is necessary to serve requirements image to fetch wheels
# during building project images
- name: Start local registry
command: "docker run -d -p {{ local_registry }}:5000 --restart always --name registry {{ local_registry_image }}"

55
playbooks/vars.yaml
View File

@@ -23,6 +23,7 @@ requirements_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "requirements"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/requirements"
tags: *tags
@@ -36,6 +37,7 @@ barbican_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "project"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/barbican"
tags: *tags
@@ -44,7 +46,7 @@ barbican_image:
- "PROJECT='barbican'"
- "PROJECT_REF={{ openstack_release }}"
- "PROFILES='fluent'"
- "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "KEEP_ALL_WHEELS=yes"
- "DIST_PACKAGES='python3-dev gcc'"
- "PIP_ARGS='--only-binary :none:'"
@@ -53,6 +55,7 @@ cinder_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "project"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/cinder"
tags: *tags
@@ -62,13 +65,14 @@ cinder_image:
- "PROJECT_REF={{ openstack_release }}"
- "PROFILES='fluent lvm ceph qemu apache'"
- "PIP_PACKAGES='python-swiftclient'"
- "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "KEEP_ALL_WHEELS=yes"
cyborg_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "project"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/cyborg"
tags: *tags
@@ -77,13 +81,14 @@ cyborg_image:
- "PROJECT='cyborg'"
- "PROJECT_REF={{ openstack_release }}"
- "DIST_PACKAGES='pciutils'"
- "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "KEEP_ALL_WHEELS=yes"
designate_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "project"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/designate"
tags: *tags
@@ -92,13 +97,14 @@ designate_image:
- "PROJECT='designate'"
- "PROJECT_REF={{ openstack_release }}"
- "PROFILES='fluent'"
- "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "KEEP_ALL_WHEELS=yes"
glance_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "project"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/glance"
tags: *tags
@@ -108,13 +114,14 @@ glance_image:
- "PROJECT_REF={{ openstack_release }}"
- "PROFILES='fluent ceph qemu-utils'"
- "PIP_PACKAGES='python-swiftclient os-brick python-cinderclient oslo-rootwrap'"
- "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "KEEP_ALL_WHEELS=yes"
heat_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "project"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/heat"
tags: *tags
@@ -123,7 +130,7 @@ heat_image:
- "PROJECT='heat'"
- "PROJECT_REF={{ openstack_release }}"
- "PROFILES='fluent apache'"
- "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "KEEP_ALL_WHEELS=yes"
- "DIST_PACKAGES='curl'"
@@ -131,6 +138,7 @@ horizon_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "project"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/horizon"
tags: *tags
@@ -139,7 +147,7 @@ horizon_image:
- "PROJECT='horizon'"
- "PROJECT_REF={{ openstack_release }}"
- "PROFILES='fluent apache'"
- "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "KEEP_ALL_WHEELS=yes"
- "PIP_PACKAGES='pymemcache'"
@@ -147,6 +155,7 @@ ironic_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "project"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/ironic"
tags: *tags
@@ -156,13 +165,14 @@ ironic_image:
- "PROJECT_REF={{ openstack_release }}"
- "PROFILES='fluent ipxe ipmi qemu tftp'"
- "DIST_PACKAGES='ethtool lshw iproute2'"
- "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "KEEP_ALL_WHEELS=yes"
keystone_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "project"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/keystone"
tags: *tags
@@ -172,13 +182,14 @@ keystone_image:
- "PROJECT_REF={{ openstack_release }}"
- "PROFILES='fluent apache ldap {{ openstack_release }}'"
- "PIP_PACKAGES='python-openstackclient'"
- "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "KEEP_ALL_WHEELS=yes"
manila_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "project"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/manila"
tags: *tags
@@ -186,13 +197,14 @@ manila_image:
- "FROM='base:{{ image_tag }}'"
- "PROJECT='manila'"
- "PROJECT_REF={{ openstack_release }}"
- "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "KEEP_ALL_WHEELS=yes"
monasca_api_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "project"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/monasca-api"
tags: *tags
@@ -202,13 +214,14 @@ monasca_api_image:
- "PROJECT_REF={{ openstack_release }}"
- "PROFILES='monasca api'"
- "PIP_PACKAGES='influxdb cassandra-driver sqlalchemy'"
- "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "KEEP_ALL_WHEELS=yes"
neutron_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "project"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/neutron"
tags: *tags
@@ -219,13 +232,14 @@ neutron_image:
- "PROFILES='fluent linuxbridge openvswitch apache vpn baremetal'"
- "PIP_PACKAGES='tap-as-a-service'"
- "DIST_PACKAGES='jq ethtool lshw'"
- "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "KEEP_ALL_WHEELS=yes"
nova_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "project"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/nova"
tags: *tags
@@ -235,13 +249,14 @@ nova_image:
- "PROJECT_REF={{ openstack_release }}"
- "PROFILES='fluent ceph linuxbridge openvswitch configdrive qemu apache migration'"
- "DIST_PACKAGES='net-tools openssh-server'"
- "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "KEEP_ALL_WHEELS=yes"
octavia_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "project"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/octavia"
tags: *tags
@@ -249,7 +264,7 @@ octavia_image:
- "FROM='base:{{ image_tag }}'"
- "PROJECT='octavia'"
- "PROJECT_REF={{ openstack_release }}"
- "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "KEEP_ALL_WHEELS=yes"
openstack_client_pip_packages:
@@ -280,6 +295,7 @@ openstack_client_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "project"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/openstack-client"
tags: *tags
@@ -287,7 +303,7 @@ openstack_client_image:
- "FROM='base:{{ image_tag }}'"
- "PROJECT='python-openstackclient'"
- "PROJECT_REF={{ openstack_release }}"
- "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "KEEP_ALL_WHEELS=yes"
- "PIP_PACKAGES='{{ openstack_client_pip_packages | join(' ') }}'"
@@ -295,6 +311,7 @@ placement_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "project"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/placement"
tags: *tags
@@ -304,13 +321,14 @@ placement_image:
- "PROJECT_REF={{ openstack_release }}"
- "PROFILES='apache'"
- "PIP_PACKAGES='httplib2'"
- "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "KEEP_ALL_WHEELS=yes"
tacker_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "project"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/tacker"
tags: *tags
@@ -318,13 +336,14 @@ tacker_image:
- "FROM='base:{{ image_tag }}'"
- "PROJECT='tacker'"
- "PROJECT_REF={{ openstack_release }}"
- "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "KEEP_ALL_WHEELS=yes"
watcher_image:
context: "."
dockerfile: "Dockerfile"
container_filename: "Dockerfile"
target: "project"
registry: "{{ image_registry.host }}"
repository: "{{ image_registry.host }}/{{ image_registry.org }}/watcher"
tags: *tags
@@ -332,5 +351,5 @@ watcher_image:
- "FROM='base:{{ image_tag }}'"
- "PROJECT='watcher'"
- "PROJECT_REF={{ openstack_release }}"
- "WHEELS='{{ local_registry }}/{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "WHEELS='{{ image_registries[0].host }}/{{ image_registries[0].org }}/requirements:{{ image_tag }}'"
- "KEEP_ALL_WHEELS=yes"

210
scripts/fetch_wheels.py
View File

@@ -1,210 +0,0 @@
#!/usr/bin/env python
import json
import os
import platform
import re
import ssl
from urllib import request as urllib2
import logging
import sys
DOCKER_REGISTRY = 'registry.hub.docker.com'
MANIFEST_V1 = 'application/vnd.oci.image.manifest.v1+json'
MANIFEST_V2 = 'application/vnd.docker.distribution.manifest.v2+json'
MANIFEST_V2_LIST = 'application/vnd.docker.distribution.manifest.list.v2+json'
ARCH_MAP = {
'x86_64': 'amd64',
'aarch64': 'arm64',
}
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
log = logging.getLogger(__name__)
def strtobool(v):
# Clone from the now-deprecated distutils
return str(v).lower() in ("yes", "true", "t", "1")
def registry_urlopen(r):
if strtobool(os.environ.get('REGISTRY_INSECURE', "False")):
resp = urllib2.urlopen(r, context=ssl._create_unverified_context())
else:
resp = urllib2.urlopen(r)
return resp
def registry_request(r, token=None):
try:
if token:
r.add_header('Authorization', 'Bearer {}'.format(token))
return registry_urlopen(r)
except urllib2.HTTPError as err:
if err.reason == 'Unauthorized' and token is None:
value = err.headers['www-authenticate'].split(' ', 2)
items = urllib2.parse_http_list(value[1])
opts = urllib2.parse_keqv_list(items)
url = "{}?service={}&scope={}".format(
opts['realm'],
opts['service'],
opts['scope']
)
auth_request = urllib2.Request(url=url)
resp = registry_urlopen(auth_request)
resp_text = resp.read().decode('utf-8').strip()
token = json.loads(resp_text)['token']
return registry_request(r, token)
raise
def get_sha(repo, tag, registry, protocol):
headers = {
'Accept': ', '.join([MANIFEST_V2_LIST, MANIFEST_V2, MANIFEST_V1])
}
url = "{}://{}/v2/{}/manifests/{}".format(protocol, registry, repo, tag)
print(url)
r = urllib2.Request(url=url, headers=headers)
resp = registry_request(r)
resp_text = resp.read().decode('utf-8').strip()
manifest = json.loads(resp_text)
if manifest['schemaVersion'] == 1:
return manifest['fsLayers'][0]['blobSum']
elif manifest['schemaVersion'] == 2:
if manifest['mediaType'] == MANIFEST_V2_LIST:
arch = platform.processor()
if arch not in ARCH_MAP:
raise SystemError("Unknown architecture: %s" % arch)
for m in manifest['manifests']:
# NOTE(mnaser): At this point, we've found the digest for the
# manifest we want, we go back and run this code
# again but getting that arch-specific manifest.
if m['platform']['architecture'] == ARCH_MAP[arch]:
tag = m['digest']
return get_sha(repo, tag, registry, protocol)
# NOTE(mnaser): If we're here, we've gone over all the manifests
# and we didn't find one that matches our requested
# architecture.
raise SystemError("Manifest does not include arch: %s" %
ARCH_MAP[arch])
else:
# NOTE(mnaser): This is the cause if the registry returns a manifest
# which isn't a list (single arch cases or getting
# a specific arch from a manifest list). The V2
# manifest orders layers from base to end (as opposed
# to V1) so we need to get the last digest.
return manifest['layers'][-1]['digest']
raise SystemError("Unable to find correct manifest schema version")
def get_blob(repo, tag, protocol, registry=DOCKER_REGISTRY):
log.info("Fetching blob from %s://%s/%s:%s", protocol, registry, repo, tag)
sha = get_sha(repo, tag, registry, protocol)
url = "{}://{}/v2/{}/blobs/{} ".format(protocol, registry, repo, sha)
print(url)
r = urllib2.Request(url=url)
resp = registry_request(r)
return resp.read()
def protocol_detection(registry, protocol='http'):
PROTOCOLS = ('http', 'https')
index = PROTOCOLS.index(protocol)
try:
url = "{}://{}".format(protocol, registry)
log.info("Trying to connect to: %s", url)
r = urllib2.Request(url)
urllib2.urlopen(r)
except (urllib2.URLError, urllib2.HTTPError) as err:
log.info("Failed to connect to %s://%s", protocol, registry)
if err.reason == 'Forbidden':
log.info("Forbidden. Protocol detected: %s", protocol)
return protocol
elif index < len(PROTOCOLS) - 1:
return protocol_detection(registry, PROTOCOLS[index + 1])
else:
raise Exception("Cannot detect protocol for registry: {} due to error: {}".format(registry, err))
else:
log.info("Protocol detected: %s", protocol)
return protocol
def get_wheels(url):
r = urllib2.Request(url=url)
resp = registry_request(r)
# Using urllib2.request.urlopen() from python3 will face the IncompleteRead and then system report connect refused.
# To avoid this problem, add an exception to ensure that all packages will be transmitted. before link down.
try:
buf = resp.read()
except Exception as e:
buf = e.partial
return buf
def parse_image(full_image):
slash_occurrences = len(re.findall('/', full_image))
repo = None
registry = DOCKER_REGISTRY
if slash_occurrences > 1:
full_image_list = full_image.split('/')
registry = full_image_list[0]
repo = '/'.join(full_image_list[1:-1])
image = full_image_list[-1]
elif slash_occurrences == 1:
repo, image = full_image.split('/')
else:
image = full_image
if image.find(':') != -1:
image, tag = image.split(':')
else:
tag = 'latest'
return registry, repo + '/' + image if repo else image, tag
def main():
if 'WHEELS' in os.environ:
wheels = os.environ['WHEELS']
else:
with open('/opt/loci/wheels', 'r') as f:
wheels = f.read()
if wheels.startswith('/'):
with open(wheels, 'rb') as f:
data = f.read()
elif wheels.startswith('http'):
data = get_wheels(wheels)
else:
registry, image, tag = parse_image(wheels)
log.info("Image parsed: wheels=%s registry=%s image=%s tag=%s",
wheels, registry, image, tag)
if os.environ.get('REGISTRY_PROTOCOL') in ['http', 'https']:
protocol = os.environ.get('REGISTRY_PROTOCOL')
elif os.environ.get('REGISTRY_PROTOCOL') == 'detect':
protocol = protocol_detection(registry)
else:
raise ValueError("Unknown protocol given in argument")
kwargs = dict()
if registry:
kwargs.update({'registry': registry})
data = get_blob(image, tag, protocol, **kwargs)
if 'WHEELS_DEST' in os.environ:
dest = os.environ['WHEELS_DEST']
else:
with open('/opt/loci/wheels', 'w') as f:
f.write(wheels)
dest = '/tmp/wheels.tar.gz'
with open(dest, 'wb') as f:
f.write(data)
if __name__ == '__main__':
main()

15
scripts/fetch_wheels.sh
View File

@@ -1,15 +0,0 @@
#!/bin/bash
set -ex
python3 $(dirname $0)/fetch_wheels.py
mkdir -p /tmp/wheels/
# Exclude all files starting with '.' as these can be control files for
# AUFS which have special meaning on AUFS backed file stores.
tar xf /tmp/wheels.tar.gz --exclude='.*' -C /tmp/wheels/
# Presence of constraint for project we build (in Stein, Train for Horizon and
# Neutron) in uc breaks project installation with unsatisfied constraints error
# This line ensures that such constraint is absent for any future occurrence
sed -i "/^${PROJECT}===/d" /tmp/wheels/upper-constraints.txt

1
scripts/install.sh
View File

@@ -63,7 +63,6 @@ if [ "${KEEP_ALL_WHEELS}" != "False" ]; then
NO_INDEX=--no-index
fi
$(dirname $0)/fetch_wheels.sh
if [[ "${PROJECT}" == "infra" ]]; then
$(dirname $0)/setup_pip.sh
$(dirname $0)/pip_install.sh bindep ${PIP_PACKAGES}

9
scripts/pip_install.sh
View File

@@ -4,4 +4,11 @@ set -ex
packages=$@
pip install --no-cache-dir --only-binary :all: --no-compile -c /tmp/wheels/global-requirements.txt -c /tmp/wheels/upper-constraints.txt --find-links /tmp/wheels/ --ignore-installed ${PIP_ARGS} ${packages}
# Presence of constraint for project we build
# in upper constraints breaks project installation
# with unsatisfied constraints error.
# This line ensures that such constraint is absent.
cp ${WHEELS_PATH}/upper-constraints.txt /tmp/upper-constraints.txt
sed -i "/^${PROJECT}===/d" /tmp/upper-constraints.txt
pip install --no-cache-dir --only-binary :all: --no-compile -c ${WHEELS_PATH}/global-requirements.txt -c /tmp/upper-constraints.txt --find-links ${WHEELS_PATH} --ignore-installed ${PIP_ARGS} ${packages}