Rewrote the build system to be more flexible / cross platform.

We can now build the CLI in a cross platform way (linux/mac/windows) all from the Makefile with relocatable VENV, DIST, and TOX directories. We can also optionally build in a docker container on platforms where docker is available.
2017-06-28 17:12:05 -07:00
parent af1672e87e
commit f51e3a614a
25 changed files with 248 additions and 200 deletions
--- a/38
+++ b/38
@@ -1,16 +1,34 @@
-all: env test packages
+all: env packages doc test
+all-docker: env-docker packages-docker doc-docker test-docker

 clean:
-	bin/clean.sh
+	@bash bin/clean.sh
+
+distclean: clean
+	@bash bin/distclean.sh

 env:
-	bin/env.sh
-
-test: env
-	bin/test.sh
-
-doc: env
-	bin/doc.sh
+	@bash bin/env.sh

 packages: env
-	bin/packages.sh
+	@bash bin/packages.sh
+
+doc: env
+	@bash bin/doc.sh
+
+test: env
+	@bash bin/test.sh
+
+env-docker:
+	@bash bin/docker.sh env
+
+packages-docker: env-docker
+	@bash bin/docker.sh packages
+
+doc-docker: env-docker
+	@bash bin/docker.sh doc
+
+test-docker: env-docker
+	@bash bin/docker.sh test
+
+.PHONY: env env-docker
--- a/bin/clean.sh
+++ b/bin/clean.sh
@@ -1,10 +1,14 @@
-#!/bin/bash -e
+#!/bin/bash

-set -o errexit -o nounset
+CURRDIR=$(dirname "${0}")
+source ${CURRDIR}/common.sh

-BASEDIR=`dirname $0`/..
-
-rm -rf $BASEDIR/.tox $BASEDIR/env $BASEDIR/dist $BASEDIR/build
-find $BASEDIR -name '*.pyc' -delete || true
+rm -rf ${BUILDDIR}/${VENV} \
+       ${BUILDDIR}/${VENV_DOCKER} \
+       ${BUILDDIR}/${TOX} \
+       ${BUILDDIR}/${TOX_DOCKER} \
+       ${BASEDIR}/build \
+       ${BASEDIR}/*.egg-info \
+       ${BASEDIR}/.coverage \
+       ${BASEDIR}/.cache
 echo "Deleted virtualenv and test artifacts."
-
--- a/bin/common.sh
+++ b/bin/common.sh
@@ -0,0 +1,27 @@
+set -o errexit -o nounset
+if [ "$(uname)" != "Windows_NT" ]; then
+  set -o pipefail
+fi
+
+: ${PROMPT:="(dcos-cli) "}
+
+: ${BUILDDIR:=$(pwd -P)}
+: ${VENV:=env}
+: ${DIST:=dist}
+: ${TOX:=.tox}
+: ${VENV_DOCKER:=env-docker}
+: ${DIST_DOCKER:=dist-docker}
+: ${TOX_DOCKER:=.tox-docker}
+
+if [ "$(uname)" = "Windows_NT" ]; then
+  BIN=Scripts
+  EXE=.exe
+  : ${PYTHON:=python${EXE}}
+  : ${VIRTUALENV:=virtualenv${EXE}}
+else
+  BIN=bin
+  EXE=
+  : ${PYTHON:=python3.5${EXE}}
+fi
+
+BASEDIR=$( cd "$(dirname $(dirname "${0}"))" > /dev/null; pwd -P )
--- a/bin/distclean.sh
+++ b/bin/distclean.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+CURRDIR=$(dirname "${0}")
+source ${CURRDIR}/common.sh
+
+rm -rf ${BUILDDIR}/${DIST} \
+       ${BUILDDIR}/${DIST_DOCKER}
+echo "Deleted distribution artifacts."
--- a/bin/doc.sh
+++ b/bin/doc.sh
@@ -1,9 +1,8 @@
-#!/bin/bash -e
+#!/bin/bash

-BASEDIR=`dirname $0`/..
+CURRDIR=$(dirname "${0}")
+source ${CURRDIR}/common.sh

-cd $BASEDIR
-source $BASEDIR/env/bin/activate
-echo "Virtualenv activated."
-
-sphinx-build -W -b html pydoc pydoc/_build
+echo "Building documentation..."
+${BUILDDIR}/${VENV}/${BIN}/sphinx-build${EXE} -W -b html pydoc pydoc/_build
+echo "Documentation built."
--- a/bin/docker.sh
+++ b/bin/docker.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+CURRDIR=$(dirname "${0}")
+source ${CURRDIR}/common.sh
+
+: ${DOCKER_RUN:="docker run \
+                     --rm \
+                     -v ${BASEDIR}:/dcos-cli \
+                     -v ${HOME}:/home/${USER} \
+                     -v /etc/passwd:/etc/passwd:ro \
+                     -v /etc/group:/etc/group:ro \
+                     -e HOME=/home/${USER} \
+                     -e VENV=${VENV_DOCKER} \
+                     -e DIST=${DIST_DOCKER} \
+                     -e TOX=${TOX_DOCKER} \
+                     -w /dcos-cli \
+                     -u $(id -u ${USER}):$(id -g ${USER}) \
+                     python:3.5"}
+
+for target in "${@}"; do
+    ${DOCKER_RUN} make ${target}
+done
--- a/bin/env.sh
+++ b/bin/env.sh
@@ -1,35 +1,52 @@
-#!/bin/bash -e
+#!/bin/bash

-BASEDIR=`dirname $0`/..
+CURRDIR=$(dirname "${0}")
+source ${CURRDIR}/common.sh

-if [ ! -d "$BASEDIR/env" ]; then
-
-    if [ ! "$(command -v pyvenv-3.5)" ]; then
-      echo "Cannot find supported python version 3.5. Exiting..."
+if [ ! -d "${BUILDDIR}/${VENV}" ]; then
+    # Check for required prerequisites.
+    echo "Checking prerequisites..."
+    if [ ! "$(command -v ${PYTHON})" ]; then
+      echo "Cannot find python. Exiting..."
      exit 1
    fi

-    pyvenv-3.5 $BASEDIR/env
-    echo "Virtualenv created."
+    PYTHON_MAJOR=$(${PYTHON} -c 'import sys; print(sys.version_info[0])')
+    PYTHON_MINOR=$(${PYTHON} -c 'import sys; print(sys.version_info[1])')

-    if [ -f "$BASEDIR/env/bin/activate" ]; then
-	    source $BASEDIR/env/bin/activate
-    else
-	    $BASEDIR/env/Scripts/activate
+    if [ "${PYTHON_MAJOR}" != "3" ] || [ "${PYTHON_MINOR}" != "5" ]; then
+        echo "Cannot find supported python version 3.5. Exiting..."
+        exit 1
    fi
-    echo "Virtualenv activated."
+    if [ "$(uname)" = "Windows_NT" ]; then
+      if [ ! "$(command -v ${VIRTUALENV})" ]; then
+          echo "Cannot find virtualenv. Exiting..."
+      fi
+    fi
+    echo "Prerequisite checks passed."

-    pip install -r $BASEDIR/requirements.txt
-    pip install -e $BASEDIR
-    echo "Requirements installed."
-
-elif [ ! -f "$BASEDIR/env/bin/activate" -o "$BASEDIR/setup.py" -nt "$BASEDIR/env/bin/activate" ]; then
-
-    source $BASEDIR/env/bin/activate
-    echo "Virtualenv activated."
-
-    pip install -r $BASEDIR/requirements.txt
-    pip install -e $BASEDIR
-    echo "Requirements installed."
+    # Create the virtualenv.
+    echo "Creating virtualenv..."
+    if [ "$(uname)" = "Windows_NT" ]; then
+      mkdir -p ${BUILDDIR}/${VENV}; cd ${BUILDDIR}/${VENV}
+      ${VIRTUALENV} --python=$(which ${PYTHON}) --prompt="${PROMPT}" --no-site-packages ${BUILDDIR}/${VENV}
+      ${VIRTUALENV} --relocatable ${BUILDDIR}/${VENV}
+      cd -
+    else
+      ${PYTHON} -m venv ${BUILDDIR}/${VENV}
+      sed -i'' -e "s#(${VENV}) #${PROMPT}#g" ${BUILDDIR}/${VENV}/${BIN}/activate
+    fi
+    echo "Virtualenv created: ${BUILDDIR}/${VENV}"

+    # Install all requirements into the virtualenv.
+    echo "Installing virtualenv requirements..."
+    ${BUILDDIR}/${VENV}/${BIN}/pip${EXE} install --upgrade pip
+    ${BUILDDIR}/${VENV}/${BIN}/pip${EXE} install -r ${BASEDIR}/requirements.txt
+    ${BUILDDIR}/${VENV}/${BIN}/pip${EXE} install -e ${BASEDIR}
+    if [ "$(uname)" = "Windows_NT" ]; then
+      ${VIRTUALENV} --relocatable ${BUILDDIR}/${VENV}
+    fi
+    echo "Virtualenv requirements installed."
+else
+    echo "Virtualenv already exists: '${BUILDDIR}/${VENV}'"
 fi
--- a/bin/packages.sh
+++ b/bin/packages.sh
@@ -1,15 +1,20 @@
-#!/bin/bash -e
+#!/bin/bash

-BASEDIR=`dirname $0`/..
-if [ -f "$BASEDIR/env/bin/python" ]; then
+CURRDIR=$(dirname "${0}")
+source ${CURRDIR}/common.sh

-	PYTHON="$BASEDIR/env/bin/python"
-else
-	PYTHON="$BASEDIR/env/Scripts/python.exe"
-fi
+cd ${BASEDIR}
+
+mkdir -p ${BUILDDIR}/${DIST}/build
+rm -rf ${BUILDDIR}/${DIST}/build/packages

 echo "Building wheel..."
-"$PYTHON" setup.py bdist_wheel
+${BUILDDIR}/${VENV}/${BIN}/python${EXE} setup.py bdist_wheel \
+    --dist-dir=${BUILDDIR}/${DIST}

 echo "Building egg..."
-"$PYTHON" setup.py sdist
+${BUILDDIR}/${VENV}/${BIN}/python${exe} setup.py sdist \
+    --dist-dir=${BUILDDIR}/${DIST}
+
+mv build ${BUILDDIR}/${DIST}/build/packages
+echo "Packages built."
--- a/bin/test.sh
+++ b/bin/test.sh
@@ -1,13 +1,8 @@
-#!/bin/bash -e
+#!/bin/bash

-BASEDIR=`dirname $0`/..
+CURRDIR=$(dirname "${0}")
+source ${CURRDIR}/common.sh

-cd $BASEDIR
-if [ -f "$BASEDIR/env/bin/activate" ]; then
-	source $BASEDIR/env/bin/activate
-else
-	$BASEDIR/env/Scripts/activate
-fi
-echo "Virtualenv activated."
-
-tox
+echo "Running tests..."
+TOXWORKDIR=${BUILDDIR}/${TOX} ${BUILDDIR}/${VENV}/${BIN}/tox${EXE}
+echo "Tests completed."
--- a/cli/Makefile
+++ b/cli/Makefile
@@ -1,22 +1,40 @@
-all: env test packages
+all: env packages binary test
+all-docker: env-docker packages-docker binary-docker test-docker

 clean:
-	bin/clean.sh
+	@bash bin/clean.sh
+
+distclean: clean
+	@bash bin/distclean.sh

 env:
-	bin/env.sh
-
-test: env
-	bin/test.sh
-
-binary: clean env packages
-	bin/create-binary.sh
-
-test-binary: binary
-	bin/test-binary.sh
-
-doc: env
-	bin/doc.sh
+	@bash bin/env.sh

 packages: env
-	bin/packages.sh
+	@bash bin/packages.sh
+
+binary: env
+	@bash bin/binary.sh
+
+test: env
+	@bash bin/test.sh
+
+test-binary: binary
+	@bash bin/test-binary.sh
+
+env-docker:
+	@bash bin/docker.sh env
+
+packages-docker: env-docker
+	@bash bin/docker.sh packages
+
+binary-docker: env-docker
+	@bash bin/docker.sh binary
+
+test-docker: env-docker
+	@bash bin/docker.sh test
+
+test-binary-docker: binary-docker
+	@bash bin/docker.sh test-binary
+
+.PHONY: env env-docker
--- a/cli/bin/binary.sh
+++ b/cli/bin/binary.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+CURRDIR=$(dirname "${0}")
+source ${CURRDIR}/common.sh
+
+echo "Building binary..."
+${BUILDDIR}/${VENV}/${BIN}/pyinstaller${EXE} \
+    --workpath=${BUILDDIR}/${DIST}/build \
+    --distpath=${BUILDDIR}/${DIST} \
+    ${BASEDIR}/binary.spec
+echo "Binary built."
--- a/cli/bin/clean.sh
+++ b/cli/bin/clean.sh
@@ -1,10 +0,0 @@
-#!/bin/bash -e
-
-set -o errexit -o nounset
-
-BASEDIR=`dirname $0`/..
-
-rm -rf $BASEDIR/.tox $BASEDIR/env $BASEDIR/dist $BASEDIR/build
-find $BASEDIR -name '*.pyc' -delete || true
-echo "Deleted virtualenv and test artifacts."
-
--- a/cli/bin/clean.sh
+++ b/cli/bin/clean.sh
@@ -0,0 +1 @@
+../../bin/clean.sh
--- a/cli/bin/common.sh
+++ b/cli/bin/common.sh
@@ -0,0 +1,5 @@
+CURRDIR=$(dirname "${0}")
+
+: ${PROMPT:="(dcos-cli/cli) "}
+
+source ${CURRDIR}/../../bin/common.sh
--- a/cli/bin/create-binary.sh
+++ b/cli/bin/create-binary.sh
@@ -1,16 +0,0 @@
-#!/bin/bash -e
-
-BASEDIR=`dirname $0`/..
-
-cd $BASEDIR
-
-if [ -f "$BASEDIR/env/bin/activate" ]; then
-        source $BASEDIR/env/bin/activate
-else
-        $BASEDIR/env/Scripts/activate
-fi
-
-pip install pyinstaller==3.1.1
-pyinstaller $BASEDIR/binary/binary.spec
-
-deactivate
--- a/cli/bin/distclean.sh
+++ b/cli/bin/distclean.sh
@@ -0,0 +1 @@
+../../bin/distclean.sh
--- a/cli/bin/doc.sh
+++ b/cli/bin/doc.sh
@@ -1,9 +0,0 @@
-#!/bin/bash -e
-
-BASEDIR=`dirname $0`/..
-
-cd $BASEDIR
-source $BASEDIR/env/bin/activate
-echo "Virtualenv activated."
-
-sphinx-build -W -b html pydoc pydoc/_build
--- a/cli/bin/docker.sh
+++ b/cli/bin/docker.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+CURRDIR=$(dirname "${0}")
+source ${CURRDIR}/common.sh
+
+: ${DOCKER_RUN:="docker run \
+               --rm \
+               -v ${BASEDIR}/..:/dcos-cli \
+               -v ${HOME}:/home/${USER} \
+               -v /etc/passwd:/etc/passwd:ro \
+               -v /etc/group:/etc/group:ro \
+               -e HOME=/home/${USER} \
+               -e VENV=${VENV_DOCKER} \
+               -e DIST=${DIST_DOCKER} \
+               -e TOX=${TOX_DOCKER} \
+               -w /dcos-cli/cli \
+               -u $(id -u ${USER}):$(id -g ${USER}) \
+               python:3.5"}
+
+source ${BASEDIR}/../bin/docker.sh
--- a/cli/bin/env.sh
+++ b/cli/bin/env.sh
@@ -1,35 +0,0 @@
-#!/bin/bash -e
-
-BASEDIR=`dirname $0`/..
-
-if [ ! -d "$BASEDIR/env" ]; then
-
-    if [ ! "$(command -v pyvenv-3.5)" ]; then
-      echo "Cannot find supported python version 3.5. Exiting..."
-      exit 1
-    fi
-
-    pyvenv-3.5 $BASEDIR/env
-    echo "Virtualenv created."
-
-    if [ -f "$BASEDIR/env/bin/activate" ]; then
-	    source $BASEDIR/env/bin/activate
-    else
-	    $BASEDIR/env/Scripts/activate
-    fi
-    echo "Virtualenv activated."
-
-    pip install -r $BASEDIR/requirements.txt
-    pip install -e $BASEDIR
-    echo "Requirements installed."
-
-elif [ ! -f "$BASEDIR/env/bin/activate" -o "$BASEDIR/setup.py" -nt "$BASEDIR/env/bin/activate" ]; then
-
-    source $BASEDIR/env/bin/activate
-    echo "Virtualenv activated."
-
-    pip install -r $BASEDIR/requirements.txt
-    pip install -e $BASEDIR
-    echo "Requirements installed."
-
-fi
--- a/cli/bin/env.sh
+++ b/cli/bin/env.sh
@@ -0,0 +1 @@
+../../bin/env.sh
--- a/cli/bin/packages.sh
+++ b/cli/bin/packages.sh
@@ -1,15 +0,0 @@
-#!/bin/bash -e
-
-BASEDIR=`dirname $0`/..
-if [ -f "$BASEDIR/env/bin/python" ]; then
-
-	PYTHON="$BASEDIR/env/bin/python"
-else
-	PYTHON="$BASEDIR/env/Scripts/python.exe"
-fi
-
-echo "Building wheel..."
-"$PYTHON" setup.py bdist_wheel
-
-echo "Building egg..."
-"$PYTHON" setup.py sdist
--- a/cli/bin/packages.sh
+++ b/cli/bin/packages.sh
@@ -0,0 +1 @@
+../../bin/packages.sh
--- a/cli/bin/test-binary.sh
+++ b/cli/bin/test-binary.sh
@@ -1,17 +1,7 @@
-#!/bin/bash -e
+#!/bin/bash

-BASEDIR=`dirname $0`/..
+CURRDIR=$(dirname "${0}")
+source ${CURRDIR}/common.sh

-cd $BASEDIR
-
-if [ -f "$BASEDIR/env/bin/activate" ]; then
-        cp tests/data/dcos.toml $DCOS_CONFIG
-        source $BASEDIR/env/bin/activate
-else
-        export DCOS_CONFIG=tests/data/dcos.toml
-        $BASEDIR/env/Scripts/activate
-fi
-
-export PATH=$BASEDIR/dist:$PATH
-py.test tests/integrations
-deactivate
+export PATH="${BUILDDIR}/${DIST}:${PATH}"
+source ${CURRDIR}/test.sh
--- a/cli/bin/test.sh
+++ b/cli/bin/test.sh
@@ -1,15 +0,0 @@
-#!/bin/bash -e
-
-BASEDIR=`dirname $0`/..
-
-cd $BASEDIR
-if [ -f "$BASEDIR/env/bin/activate" ]; then
-	source $BASEDIR/env/bin/activate
-else
-	$BASEDIR/env/Scripts/activate
-fi
-echo "Virtualenv activated."
-
-chmod 600 $BASEDIR/tests/data/dcos.toml
-
-tox
--- a/cli/bin/test.sh
+++ b/cli/bin/test.sh
@@ -0,0 +1 @@
+../../bin/test.sh
--- a/cli/requirements.txt
+++ b/cli/requirements.txt
@@ -3,4 +3,5 @@ tox>=2.2, <3.0
 wheel>=0.24.0, <1.0
 pytest>=2.9.1
 teamcity-messages>=1.20
+PyInstaller==3.2.1
 -e .. # Install the DC/OS package
--- a/cli/tox.ini
+++ b/cli/tox.ini
@@ -1,5 +1,6 @@
 [tox]
 envlist = py35-syntax, py35-unit, py35-integration
+toxworkdir={env:TOXWORKDIR:.tox}

 [flake8]
 application-import-names=dcos,dcoscli
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,6 @@
 [tox]
 envlist = py35-syntax, py35-unit
+toxworkdir={env:TOXWORKDIR:.tox}

 [flake8]
 application-import-names=dcos
--- a/tox.win.ini
+++ b/tox.win.ini
@@ -1,5 +1,6 @@
 [tox]
 envlist = py35-unit
+toxworkdir={env:TOXWORKDIR:.tox}

 [testenv]
 setenv =