Resolves installation problems in Docker

A recent change to implement Pipenv caused VCS dependencies (Promenade,
Deckhand, and Shipyard) to not be fully installed in Docker images. This
change removes the "editable" tags from the VCS dependencies to ensure a
full install as having "editable" enabled will only install dependencies
in development mode.

Unfortunately, the "editable" tag is required to install the
requirements.txt for a VCS dependency. To get the lower-level
dependencies installed from VCS dependencies, I implemented a few
commands in the Dockerfiles to retrieve the appropriate requirements.txt
and install them before fully installing Pegleg. An upcoming release of
Pipenv will fix the existing problems with VCS dependency resolution at
which point this temporary solution may be removed.

Adds manual installation of VCS dependency requirements.txt in Docker

Removes "editable" tags from VCS dependencies

Moves docker package to deployment packages from dev packages

Adds .env file to track VCS refs used by Docker for requirements.txt

Change-Id: Ifdb1fe960b32280dcb3c5308e56b2d608f848975
This commit is contained in:
Ian H Pittwood 2019-08-15 09:02:02 -05:00 committed by Ian H. Pittwood
parent b93363e753
commit 36b8e9fe60
8 changed files with 135 additions and 878 deletions

View File

@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
include vcs-requirements.env
PEGLEG_BUILD_CTX ?= pegleg PEGLEG_BUILD_CTX ?= pegleg
IMAGE_NAME ?= pegleg IMAGE_NAME ?= pegleg
IMAGE_PREFIX ?= airshipit IMAGE_PREFIX ?= airshipit
@ -75,7 +77,7 @@ ifeq ($(USE_PROXY), true)
--label "org.opencontainers.image.revision=$(COMMIT)" \ --label "org.opencontainers.image.revision=$(COMMIT)" \
--label "org.opencontainers.image.created=$(shell date --rfc-3339=seconds --utc)" \ --label "org.opencontainers.image.created=$(shell date --rfc-3339=seconds --utc)" \
--label "org.opencontainers.image.title=$(IMAGE_NAME)" \ --label "org.opencontainers.image.title=$(IMAGE_NAME)" \
-f images/pegleg/Dockerfile.$(DISTRO) \ -f images/pegleg/Dockerfile.${DISTRO} \
$(_BASE_IMAGE_ARG) \ $(_BASE_IMAGE_ARG) \
--build-arg http_proxy=$(PROXY) \ --build-arg http_proxy=$(PROXY) \
--build-arg https_proxy=$(PROXY) \ --build-arg https_proxy=$(PROXY) \
@ -83,15 +85,21 @@ ifeq ($(USE_PROXY), true)
--build-arg HTTPS_PROXY=$(PROXY) \ --build-arg HTTPS_PROXY=$(PROXY) \
--build-arg no_proxy=$(NO_PROXY) \ --build-arg no_proxy=$(NO_PROXY) \
--build-arg NO_PROXY=$(NO_PROXY) \ --build-arg NO_PROXY=$(NO_PROXY) \
--build-arg ctx_base=$(PEGLEG_BUILD_CTX) . --build-arg ctx_base=$(PEGLEG_BUILD_CTX) . \
--build-arg DECKHAND_VERSION=${DECKHAND_VERSION} \
--build-arg PROMENADE_VERSION=${PROMENADE_VERSION} \
--build-arg SHIPYARD_VERSION=${SHIPYARD_VERSION}
else else
docker build -t $(IMAGE) --network=host --label $(LABEL) \ docker build -t $(IMAGE) --network=host --label $(LABEL) \
--label "org.opencontainers.image.revision=$(COMMIT)" \ --label "org.opencontainers.image.revision=$(COMMIT)" \
--label "org.opencontainers.image.created=$(shell date --rfc-3339=seconds --utc)" \ --label "org.opencontainers.image.created=$(shell date --rfc-3339=seconds --utc)" \
--label "org.opencontainers.image.title=$(IMAGE_NAME)" \ --label "org.opencontainers.image.title=$(IMAGE_NAME)" \
-f images/pegleg/Dockerfile.$(DISTRO) \ -f images/pegleg/Dockerfile.${DISTRO} \
$(_BASE_IMAGE_ARG) \ $(_BASE_IMAGE_ARG) \
--build-arg ctx_base=$(PEGLEG_BUILD_CTX) . --build-arg ctx_base=$(PEGLEG_BUILD_CTX) . \
--build-arg DECKHAND_VERSION=${DECKHAND_VERSION} \
--build-arg PROMENADE_VERSION=${PROMENADE_VERSION} \
--build-arg SHIPYARD_VERSION=${SHIPYARD_VERSION}
endif endif
ifeq ($(PUSH_IMAGE), true) ifeq ($(PUSH_IMAGE), true)
docker push $(IMAGE) docker push $(IMAGE)

View File

@ -16,7 +16,6 @@ yapf = ">=0.27.0"
hacking = ">=1.1.0" hacking = ">=1.1.0"
flake8-import-order = ">=0.18.1" flake8-import-order = ">=0.18.1"
bandit = ">=1.6.0" bandit = ">=1.6.0"
docker = ">=3.7.2"
[packages] [packages]
click = ">=6.7" click = ">=6.7"
@ -24,10 +23,11 @@ jsonschema = "~=2.6.0"
cryptography = "~=2.3.1" cryptography = "~=2.3.1"
python-dateutil = "~=2.7.3" python-dateutil = "~=2.7.3"
GitPython = "~=2.1.11" GitPython = "~=2.1.11"
docker = ">=3.7.2"
PyYAML = "~=5.1" PyYAML = "~=5.1"
deckhand = {git = "https://opendev.org/airship/deckhand.git",ref = "a4850f8c4fc02dd1971282c6d9e4beb9cd5be320",editable = true} deckhand = {git = "https://opendev.org/airship/deckhand.git",ref = "a4850f8c4fc02dd1971282c6d9e4beb9cd5be320"}
shipyard-client = {git = "https://opendev.org/airship/shipyard.git",ref = "1472a22dce698ea77f15b007906b263eef470f52",subdirectory = "src/bin/shipyard_client",editable = true} shipyard-client = {git = "https://opendev.org/airship/shipyard.git",ref = "1472a22dce698ea77f15b007906b263eef470f52",subdirectory = "src/bin/shipyard_client"}
promenade = {git = "https://opendev.org/airship/promenade.git",ref = "89f9842b2ec428709a05fd0a8d4cd1796043375d",editable = true} promenade = {git = "https://opendev.org/airship/promenade.git",ref = "89f9842b2ec428709a05fd0a8d4cd1796043375d"}
oslo-utils = "~=3.41.0" oslo-utils = "~=3.41.0"
[requires] [requires]

852
Pipfile.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -11,6 +11,9 @@ LABEL org.opencontainers.image.licenses='Apache-2.0'
ENV LANG=C.UTF-8 ENV LANG=C.UTF-8
ENV LC_ALL=C.UTF-8 ENV LC_ALL=C.UTF-8
ARG DECKHAND_VERSION=branch/master
ARG SHIPYARD_VERSION=branch/master
ARG PROMENADE_VERSION=branch/master
RUN set -x \ RUN set -x \
&& zypper up -y \ && zypper up -y \
@ -38,7 +41,10 @@ VOLUME /var/pegleg
WORKDIR /var/pegleg WORKDIR /var/pegleg
COPY requirements.txt /opt/pegleg/requirements.txt COPY requirements.txt /opt/pegleg/requirements.txt
RUN pip3 install --no-cache-dir -r /opt/pegleg/requirements.txt RUN pip3 install -r https://opendev.org/airship/deckhand/raw/${DECKHAND_VERSION}/requirements.txt \
&& pip3 install -r https://opendev.org/airship/promenade/raw/${PROMENADE_VERSION}/requirements.txt \
&& pip3 install -r https://opendev.org/airship/shipyard/raw/${SHIPYARD_VERSION}/src/bin/shipyard_client/requirements.txt \
&& pip3 install --no-cache-dir -r /opt/pegleg/requirements.txt
COPY tools/install-cfssl.sh /opt/pegleg/tools/install-cfssl.sh COPY tools/install-cfssl.sh /opt/pegleg/tools/install-cfssl.sh
RUN /opt/pegleg/tools/install-cfssl.sh ${CFSSLURL} RUN /opt/pegleg/tools/install-cfssl.sh ${CFSSLURL}

View File

@ -11,6 +11,9 @@ LABEL org.opencontainers.image.licenses='Apache-2.0'
ENV LANG=C.UTF-8 ENV LANG=C.UTF-8
ENV LC_ALL=C.UTF-8 ENV LC_ALL=C.UTF-8
ARG DECKHAND_VERSION=branch/master
ARG SHIPYARD_VERSION=branch/master
ARG PROMENADE_VERSION=branch/master
RUN set -ex \ RUN set -ex \
&& apt-get update -qq \ && apt-get update -qq \
@ -41,7 +44,10 @@ VOLUME /var/pegleg
WORKDIR /var/pegleg WORKDIR /var/pegleg
COPY requirements.txt /opt/pegleg/requirements.txt COPY requirements.txt /opt/pegleg/requirements.txt
RUN pip3 install --no-cache-dir -r /opt/pegleg/requirements.txt RUN pip3 install -r https://opendev.org/airship/deckhand/raw/${DECKHAND_VERSION}/requirements.txt \
&& pip3 install -r https://opendev.org/airship/promenade/raw/${PROMENADE_VERSION}/requirements.txt \
&& pip3 install -r https://opendev.org/airship/shipyard/raw/${SHIPYARD_VERSION}/src/bin/shipyard_client/requirements.txt \
&& pip3 install --no-cache-dir -r /opt/pegleg/requirements.txt
COPY tools/install-cfssl.sh /opt/pegleg/tools/install-cfssl.sh COPY tools/install-cfssl.sh /opt/pegleg/tools/install-cfssl.sh
RUN /opt/pegleg/tools/install-cfssl.sh ${CFSSLURL} RUN /opt/pegleg/tools/install-cfssl.sh ${CFSSLURL}

View File

@ -11,6 +11,9 @@ LABEL org.opencontainers.image.licenses='Apache-2.0'
ENV LANG=C.UTF-8 ENV LANG=C.UTF-8
ENV LC_ALL=C.UTF-8 ENV LC_ALL=C.UTF-8
ARG DECKHAND_VERSION=branch/master
ARG SHIPYARD_VERSION=branch/master
ARG PROMENADE_VERSION=branch/master
RUN set -ex \ RUN set -ex \
&& apt-get update -qq \ && apt-get update -qq \
@ -41,7 +44,10 @@ VOLUME /var/pegleg
WORKDIR /var/pegleg WORKDIR /var/pegleg
COPY requirements.txt /opt/pegleg/requirements.txt COPY requirements.txt /opt/pegleg/requirements.txt
RUN pip3 install --no-cache-dir -r /opt/pegleg/requirements.txt RUN pip3 install -r https://opendev.org/airship/deckhand/raw/${DECKHAND_VERSION}/requirements.txt \
&& pip3 install -r https://opendev.org/airship/promenade/raw/${PROMENADE_VERSION}/requirements.txt \
&& pip3 install -r https://opendev.org/airship/shipyard/raw/${SHIPYARD_VERSION}/src/bin/shipyard_client/requirements.txt \
&& pip3 install --no-cache-dir -r /opt/pegleg/requirements.txt
COPY tools/install-cfssl.sh /opt/pegleg/tools/install-cfssl.sh COPY tools/install-cfssl.sh /opt/pegleg/tools/install-cfssl.sh
RUN /opt/pegleg/tools/install-cfssl.sh ${CFSSLURL} RUN /opt/pegleg/tools/install-cfssl.sh ${CFSSLURL}

View File

@ -1,118 +1,34 @@
-i https://pypi.org/simple -i https://pypi.org/simple
-e git+https://opendev.org/airship/deckhand.git@a4850f8c4fc02dd1971282c6d9e4beb9cd5be320#egg=deckhand
-e git+https://opendev.org/airship/promenade.git@89f9842b2ec428709a05fd0a8d4cd1796043375d#egg=promenade
-e git+https://opendev.org/airship/shipyard.git@1472a22dce698ea77f15b007906b263eef470f52#egg=shipyard-client&subdirectory=src/bin/shipyard_client
alembic==1.0.1
amqp==2.5.0
argparse==1.4.0
arrow==0.12.1
asn1crypto==0.24.0 asn1crypto==0.24.0
attrs==19.1.0
babel==2.7.0 babel==2.7.0
beaker==1.10.0
cachetools==3.1.1
certifi==2019.6.16 certifi==2019.6.16
cffi==1.12.3 cffi==1.12.3
chardet==3.0.4 chardet==3.0.4
click-default-group==1.2 click==7.0
click==6.7
cliff==2.15.0
cmd2==0.9.16 ; python_version >= '3.0'
colorama==0.4.1
cryptography==2.3.1 cryptography==2.3.1
debtcollector==1.21.0 debtcollector==1.21.0
decorator==4.4.0 docker==4.0.2
deepdiff==3.3.0 git+https://opendev.org/airship/deckhand.git@a4850f8c4fc02dd1971282c6d9e4beb9cd5be320#egg=deckhand
dnspython==1.16.0 git+https://opendev.org/airship/promenade.git@89f9842b2ec428709a05fd0a8d4cd1796043375d#egg=promenade
dogpile.cache==0.7.1 git+https://opendev.org/airship/shipyard.git@1472a22dce698ea77f15b007906b263eef470f52#egg=shipyard-client&subdirectory=src/bin/shipyard_client
eventlet==0.25.0
extras==1.0.0
falcon==1.4.1
fasteners==0.15
fixtures==3.0.0
flake8==2.6.2
futurist==1.8.1
gitdb2==2.0.5 gitdb2==2.0.5
gitpython==2.1.13 gitpython==2.1.14
greenlet==0.4.15 idna==2.8
hacking==1.1.0
idna==2.7
iso8601==0.1.12 iso8601==0.1.12
jinja2==2.10.1
jsonpath-ng==1.4.3
jsonpickle==1.2
jsonschema==2.6.0 jsonschema==2.6.0
keystoneauth1==3.11.1
keystonemiddleware==5.3.0
kombu==4.6.3
linecache2==1.0.0
mako==1.1.0
markupsafe==1.1.1
mccabe==0.5.3
monotonic==1.5
msgpack==0.6.1
netaddr==0.7.19 netaddr==0.7.19
netifaces==0.10.9 netifaces==0.10.9
networkx==2.2
os-service-types==1.7.0
oslo-utils==3.41.0 oslo-utils==3.41.0
oslo.cache==1.31.1
oslo.concurrency==3.28.1
oslo.config==6.6.2
oslo.context==2.22.1
oslo.db==4.41.1
oslo.i18n==3.23.1 oslo.i18n==3.23.1
oslo.log==3.40.1 pbr==5.4.2
oslo.messaging==9.1.1
oslo.middleware==3.36.0
oslo.policy==1.40.1
oslo.serialization==2.28.1
oslo.service==1.40.0
oslo.utils==3.40.2
paste==3.0.1
pastedeploy==1.5.2
pbr==5.1.0
ply==3.11
prettytable==0.7.2
psycopg2==2.7.5
ptable==0.9.2
pycadf==2.9.0
pycodestyle==2.0.0
pycparser==2.19 pycparser==2.19
pyflakes==1.2.3
pyinotify==0.9.6 ; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5'
pyparsing==2.4.2 pyparsing==2.4.2
pyperclip==1.7.0
python-barbicanclient==4.7.0
python-dateutil==2.7.5 python-dateutil==2.7.5
python-editor==1.0.4
python-keystoneclient==3.18.0
python-memcached==1.59
python-mimeparse==1.6.0
pytz==2019.2 pytz==2019.2
pyyaml==5.1 pyyaml==5.1.2
repoze.lru==0.7 requests==2.22.0
requests==2.20.0 six==1.12.0
rfc3986==1.3.2
routes==2.4.1
six==1.11.0
smmap2==2.0.5 smmap2==2.0.5
sqlalchemy-migrate==0.12.0 urllib3==1.25.3
sqlalchemy==1.3.6 websocket-client==0.56.0
sqlparse==0.3.0
statsd==3.3.0
stevedore==1.30.0
tempita==0.5.2
testresources==2.0.1
testscenarios==0.5.0
testtools==2.3.0
traceback2==1.4.0
unittest2==1.1.0
urllib3==1.24.3
uwsgi==2.0.17.1
vine==1.3.0
wcwidth==0.1.7
webob==1.8.5
werkzeug==0.14.1
wrapt==1.11.2 wrapt==1.11.2
yappi==1.0

3
vcs-requirements.env Normal file
View File

@ -0,0 +1,3 @@
DECKHAND_VERSION=commit/a4850f8c4fc02dd1971282c6d9e4beb9cd5be320
SHIPYARD_VERSION=commit/1472a22dce698ea77f15b007906b263eef470f52
PROMENADE_VERSION=commit/89f9842b2ec428709a05fd0a8d4cd1796043375d