Resolves installation problems in Docker

A recent change to implement Pipenv caused VCS dependencies (Promenade,
Deckhand, and Shipyard) to not be fully installed in Docker images. This
change removes the "editable" tags from the VCS dependencies to ensure a
full install as having "editable" enabled will only install dependencies
in development mode.

Unfortunately, the "editable" tag is required to install the
requirements.txt for a VCS dependency. To get the lower-level
dependencies installed from VCS dependencies, I implemented a few
commands in the Dockerfiles to retrieve the appropriate requirements.txt
and install them before fully installing Pegleg. An upcoming release of
Pipenv will fix the existing problems with VCS dependency resolution at
which point this temporary solution may be removed.

Adds manual installation of VCS dependency requirements.txt in Docker

Removes "editable" tags from VCS dependencies

Moves docker package to deployment packages from dev packages

Adds .env file to track VCS refs used by Docker for requirements.txt

Change-Id: Ifdb1fe960b32280dcb3c5308e56b2d608f848975

Makefile

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+include vcs-requirements.env
+
 PEGLEG_BUILD_CTX  ?= pegleg
 IMAGE_NAME        ?= pegleg
 IMAGE_PREFIX      ?= airshipit
@@ -75,7 +77,7 @@ ifeq ($(USE_PROXY), true)
 		--label "org.opencontainers.image.revision=$(COMMIT)" \
 		--label "org.opencontainers.image.created=$(shell date --rfc-3339=seconds --utc)" \
 		--label "org.opencontainers.image.title=$(IMAGE_NAME)" \
-		-f images/pegleg/Dockerfile.$(DISTRO) \
+		-f images/pegleg/Dockerfile.${DISTRO} \
 		$(_BASE_IMAGE_ARG) \
 		--build-arg http_proxy=$(PROXY) \
 		--build-arg https_proxy=$(PROXY) \
@@ -83,15 +85,21 @@ ifeq ($(USE_PROXY), true)
 		--build-arg HTTPS_PROXY=$(PROXY) \
 		--build-arg no_proxy=$(NO_PROXY) \
 		--build-arg NO_PROXY=$(NO_PROXY) \
-		--build-arg ctx_base=$(PEGLEG_BUILD_CTX) .
+		--build-arg ctx_base=$(PEGLEG_BUILD_CTX) . \
+		--build-arg DECKHAND_VERSION=${DECKHAND_VERSION} \
+		--build-arg PROMENADE_VERSION=${PROMENADE_VERSION} \
+		--build-arg SHIPYARD_VERSION=${SHIPYARD_VERSION}
 else
 	docker build -t $(IMAGE) --network=host --label $(LABEL) \
 		--label "org.opencontainers.image.revision=$(COMMIT)" \
 		--label "org.opencontainers.image.created=$(shell date --rfc-3339=seconds --utc)" \
 		--label "org.opencontainers.image.title=$(IMAGE_NAME)" \
-		-f images/pegleg/Dockerfile.$(DISTRO) \
+		-f images/pegleg/Dockerfile.${DISTRO} \
 		$(_BASE_IMAGE_ARG) \
-		--build-arg ctx_base=$(PEGLEG_BUILD_CTX) .
+		--build-arg ctx_base=$(PEGLEG_BUILD_CTX) . \
+		--build-arg DECKHAND_VERSION=${DECKHAND_VERSION} \
+		--build-arg PROMENADE_VERSION=${PROMENADE_VERSION} \
+		--build-arg SHIPYARD_VERSION=${SHIPYARD_VERSION}
 endif
 ifeq ($(PUSH_IMAGE), true)
 	docker push $(IMAGE)

Pipfile

@@ -16,7 +16,6 @@ yapf = ">=0.27.0"
 hacking = ">=1.1.0"
 flake8-import-order = ">=0.18.1"
 bandit = ">=1.6.0"
-docker = ">=3.7.2"
 
 [packages]
 click = ">=6.7"
@@ -24,10 +23,11 @@ jsonschema = "~=2.6.0"
 cryptography = "~=2.3.1"
 python-dateutil = "~=2.7.3"
 GitPython = "~=2.1.11"
+docker = ">=3.7.2"
 PyYAML = "~=5.1"
-deckhand = {git = "https://opendev.org/airship/deckhand.git",ref = "a4850f8c4fc02dd1971282c6d9e4beb9cd5be320",editable = true}
+deckhand = {git = "https://opendev.org/airship/deckhand.git",ref = "a4850f8c4fc02dd1971282c6d9e4beb9cd5be320"}
-shipyard-client = {git = "https://opendev.org/airship/shipyard.git",ref = "1472a22dce698ea77f15b007906b263eef470f52",subdirectory = "src/bin/shipyard_client",editable = true}
+shipyard-client = {git = "https://opendev.org/airship/shipyard.git",ref = "1472a22dce698ea77f15b007906b263eef470f52",subdirectory = "src/bin/shipyard_client"}
-promenade = {git = "https://opendev.org/airship/promenade.git",ref = "89f9842b2ec428709a05fd0a8d4cd1796043375d",editable = true}
+promenade = {git = "https://opendev.org/airship/promenade.git",ref = "89f9842b2ec428709a05fd0a8d4cd1796043375d"}
 oslo-utils = "~=3.41.0"
 
 [requires]

images/pegleg/Dockerfile.opensuse_15

@@ -11,6 +11,9 @@ LABEL org.opencontainers.image.licenses='Apache-2.0'
 
 ENV LANG=C.UTF-8
 ENV LC_ALL=C.UTF-8
+ARG DECKHAND_VERSION=branch/master
+ARG SHIPYARD_VERSION=branch/master
+ARG PROMENADE_VERSION=branch/master
 
 RUN set -x \
     && zypper up -y \
@@ -38,7 +41,10 @@ VOLUME /var/pegleg
 WORKDIR /var/pegleg
 
 COPY requirements.txt /opt/pegleg/requirements.txt
-RUN pip3 install --no-cache-dir -r /opt/pegleg/requirements.txt
+RUN pip3 install -r https://opendev.org/airship/deckhand/raw/${DECKHAND_VERSION}/requirements.txt \
+    && pip3 install -r https://opendev.org/airship/promenade/raw/${PROMENADE_VERSION}/requirements.txt \
+    && pip3 install -r https://opendev.org/airship/shipyard/raw/${SHIPYARD_VERSION}/src/bin/shipyard_client/requirements.txt \
+    && pip3 install --no-cache-dir -r /opt/pegleg/requirements.txt
 
 COPY tools/install-cfssl.sh /opt/pegleg/tools/install-cfssl.sh
 RUN /opt/pegleg/tools/install-cfssl.sh ${CFSSLURL}

images/pegleg/Dockerfile.ubuntu_bionic

@@ -11,6 +11,9 @@ LABEL org.opencontainers.image.licenses='Apache-2.0'
 
 ENV LANG=C.UTF-8
 ENV LC_ALL=C.UTF-8
+ARG DECKHAND_VERSION=branch/master
+ARG SHIPYARD_VERSION=branch/master
+ARG PROMENADE_VERSION=branch/master
 
 RUN set -ex \
     && apt-get update -qq \
@@ -41,7 +44,10 @@ VOLUME /var/pegleg
 WORKDIR /var/pegleg
 
 COPY requirements.txt /opt/pegleg/requirements.txt
-RUN pip3 install --no-cache-dir -r /opt/pegleg/requirements.txt
+RUN pip3 install -r https://opendev.org/airship/deckhand/raw/${DECKHAND_VERSION}/requirements.txt \
+    && pip3 install -r https://opendev.org/airship/promenade/raw/${PROMENADE_VERSION}/requirements.txt \
+    && pip3 install -r https://opendev.org/airship/shipyard/raw/${SHIPYARD_VERSION}/src/bin/shipyard_client/requirements.txt \
+    && pip3 install --no-cache-dir -r /opt/pegleg/requirements.txt
 
 COPY tools/install-cfssl.sh /opt/pegleg/tools/install-cfssl.sh
 RUN /opt/pegleg/tools/install-cfssl.sh ${CFSSLURL}

images/pegleg/Dockerfile.ubuntu_xenial

@@ -11,6 +11,9 @@ LABEL org.opencontainers.image.licenses='Apache-2.0'
 
 ENV LANG=C.UTF-8
 ENV LC_ALL=C.UTF-8
+ARG DECKHAND_VERSION=branch/master
+ARG SHIPYARD_VERSION=branch/master
+ARG PROMENADE_VERSION=branch/master
 
 RUN set -ex \
     && apt-get update -qq \
@@ -41,7 +44,10 @@ VOLUME /var/pegleg
 WORKDIR /var/pegleg
 
 COPY requirements.txt /opt/pegleg/requirements.txt
-RUN pip3 install --no-cache-dir -r /opt/pegleg/requirements.txt
+RUN pip3 install -r https://opendev.org/airship/deckhand/raw/${DECKHAND_VERSION}/requirements.txt \
+    && pip3 install -r https://opendev.org/airship/promenade/raw/${PROMENADE_VERSION}/requirements.txt \
+    && pip3 install -r https://opendev.org/airship/shipyard/raw/${SHIPYARD_VERSION}/src/bin/shipyard_client/requirements.txt \
+    && pip3 install --no-cache-dir -r /opt/pegleg/requirements.txt
 
 COPY tools/install-cfssl.sh /opt/pegleg/tools/install-cfssl.sh
 RUN /opt/pegleg/tools/install-cfssl.sh ${CFSSLURL}

requirements.txt

@@ -1,118 +1,34 @@
 -i https://pypi.org/simple
--e git+https://opendev.org/airship/deckhand.git@a4850f8c4fc02dd1971282c6d9e4beb9cd5be320#egg=deckhand
--e git+https://opendev.org/airship/promenade.git@89f9842b2ec428709a05fd0a8d4cd1796043375d#egg=promenade
--e git+https://opendev.org/airship/shipyard.git@1472a22dce698ea77f15b007906b263eef470f52#egg=shipyard-client&subdirectory=src/bin/shipyard_client
-alembic==1.0.1
-amqp==2.5.0
-argparse==1.4.0
-arrow==0.12.1
 asn1crypto==0.24.0
-attrs==19.1.0
 babel==2.7.0
-beaker==1.10.0
-cachetools==3.1.1
 certifi==2019.6.16
 cffi==1.12.3
 chardet==3.0.4
-click-default-group==1.2
+click==7.0
-click==6.7
-cliff==2.15.0
-cmd2==0.9.16 ; python_version >= '3.0'
-colorama==0.4.1
 cryptography==2.3.1
 debtcollector==1.21.0
-decorator==4.4.0
+docker==4.0.2
-deepdiff==3.3.0
+git+https://opendev.org/airship/deckhand.git@a4850f8c4fc02dd1971282c6d9e4beb9cd5be320#egg=deckhand
-dnspython==1.16.0
+git+https://opendev.org/airship/promenade.git@89f9842b2ec428709a05fd0a8d4cd1796043375d#egg=promenade
-dogpile.cache==0.7.1
+git+https://opendev.org/airship/shipyard.git@1472a22dce698ea77f15b007906b263eef470f52#egg=shipyard-client&subdirectory=src/bin/shipyard_client
-eventlet==0.25.0
-extras==1.0.0
-falcon==1.4.1
-fasteners==0.15
-fixtures==3.0.0
-flake8==2.6.2
-futurist==1.8.1
 gitdb2==2.0.5
-gitpython==2.1.13
+gitpython==2.1.14
-greenlet==0.4.15
+idna==2.8
-hacking==1.1.0
-idna==2.7
 iso8601==0.1.12
-jinja2==2.10.1
-jsonpath-ng==1.4.3
-jsonpickle==1.2
 jsonschema==2.6.0
-keystoneauth1==3.11.1
-keystonemiddleware==5.3.0
-kombu==4.6.3
-linecache2==1.0.0
-mako==1.1.0
-markupsafe==1.1.1
-mccabe==0.5.3
-monotonic==1.5
-msgpack==0.6.1
 netaddr==0.7.19
 netifaces==0.10.9
-networkx==2.2
-os-service-types==1.7.0
 oslo-utils==3.41.0
-oslo.cache==1.31.1
-oslo.concurrency==3.28.1
-oslo.config==6.6.2
-oslo.context==2.22.1
-oslo.db==4.41.1
 oslo.i18n==3.23.1
-oslo.log==3.40.1
+pbr==5.4.2
-oslo.messaging==9.1.1
-oslo.middleware==3.36.0
-oslo.policy==1.40.1
-oslo.serialization==2.28.1
-oslo.service==1.40.0
-oslo.utils==3.40.2
-paste==3.0.1
-pastedeploy==1.5.2
-pbr==5.1.0
-ply==3.11
-prettytable==0.7.2
-psycopg2==2.7.5
-ptable==0.9.2
-pycadf==2.9.0
-pycodestyle==2.0.0
 pycparser==2.19
-pyflakes==1.2.3
-pyinotify==0.9.6 ; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5'
 pyparsing==2.4.2
-pyperclip==1.7.0
-python-barbicanclient==4.7.0
 python-dateutil==2.7.5
-python-editor==1.0.4
-python-keystoneclient==3.18.0
-python-memcached==1.59
-python-mimeparse==1.6.0
 pytz==2019.2
-pyyaml==5.1
+pyyaml==5.1.2
-repoze.lru==0.7
+requests==2.22.0
-requests==2.20.0
+six==1.12.0
-rfc3986==1.3.2
-routes==2.4.1
-six==1.11.0
 smmap2==2.0.5
-sqlalchemy-migrate==0.12.0
+urllib3==1.25.3
-sqlalchemy==1.3.6
+websocket-client==0.56.0
-sqlparse==0.3.0
-statsd==3.3.0
-stevedore==1.30.0
-tempita==0.5.2
-testresources==2.0.1
-testscenarios==0.5.0
-testtools==2.3.0
-traceback2==1.4.0
-unittest2==1.1.0
-urllib3==1.24.3
-uwsgi==2.0.17.1
-vine==1.3.0
-wcwidth==0.1.7
-webob==1.8.5
-werkzeug==0.14.1
 wrapt==1.11.2
-yappi==1.0

vcs-requirements.env

@@ -0,0 +1,3 @@
+DECKHAND_VERSION=commit/a4850f8c4fc02dd1971282c6d9e4beb9cd5be320
+SHIPYARD_VERSION=commit/1472a22dce698ea77f15b007906b263eef470f52
+PROMENADE_VERSION=commit/89f9842b2ec428709a05fd0a8d4cd1796043375d