From f77d86cc681ace37e582352c04415af00c1224c8 Mon Sep 17 00:00:00 2001 From: sslypushenko Date: Fri, 8 May 2015 19:23:26 +0300 Subject: [PATCH] Local Refstack in Docker Main purpose of this path is providing a way how to easily create local env of Refstack API with from your latest code in Docker container. I should be helpful for testing new features and newcomers developers. run-in-docker [OPTIONS] [COMMAND] - run refstack container (if it is not running), upload latest project code into container and run Refstack API in it (default COMMAND is 'api-up'). Just run ./drun-in-docker, wait untill it finish and then check it on https://127.0.0.1 It is important to set env[REFSTACK_HOST] with public host for your local API. By default 127.0.0.1 is used, should work fine if you access to your local Refstack only from your localhost. You can customize Refstack API config with editing docker/templates/refstack.conf.tmpl. It is a bash template. You can use ${SOME_ENV_VARIABLE} in it. Available options: -r Force delete '${CONTAINER}' container and run it again -i Run container with isolated MySQL data. By default MySQL data stores in refstack_data_DATA-BASE-REVISON container It reuses if such container exists. If you want to drop DB data, just execute sudo docker rm refstack_data_DATA-BASE-REVISON -b Force delete '${IMAGE}' image and built it ag -d Turn on debug information -h Print usage message In-container commands: api-up - sync project and run Refstack API api-init-db - initialize Refstack database api-db-version - get current migration version of Refstack database api-sync - sync project files in contaner with project on host activate - activate python virtual env mysql - open mysql console Requirements: Docker 1.6 (How to update on Ubuntu http://www.ubuntuupdates.org/ppa/docker) Change-Id: I26422aecaf68af6c340ebcc2a8a36d2a4907d84c --- .dockerignore | 12 +++ __init__.py | 0 docker/Dockerfile | 45 ++++++++ docker/nginx/refstack-site.conf | 19 ++++ docker/nginx/refstack_dev.crt | 17 ++++ docker/nginx/refstack_dev.key | 15 +++ docker/scripts/api-db-version | 2 + docker/scripts/api-init-db | 5 + docker/scripts/api-sync | 5 + docker/scripts/api-up | 5 + docker/scripts/start.sh | 78 ++++++++++++++ docker/templates/config.json.tmpl | 1 + docker/templates/refstack.conf.tmpl | 12 +++ refstack/api/app.py | 6 +- requirements.txt | 2 +- run-in-docker | 153 ++++++++++++++++++++++++++++ 16 files changed, 375 insertions(+), 2 deletions(-) create mode 100644 .dockerignore delete mode 100644 __init__.py create mode 100644 docker/Dockerfile create mode 100644 docker/nginx/refstack-site.conf create mode 100644 docker/nginx/refstack_dev.crt create mode 100644 docker/nginx/refstack_dev.key create mode 100755 docker/scripts/api-db-version create mode 100755 docker/scripts/api-init-db create mode 100755 docker/scripts/api-sync create mode 100755 docker/scripts/api-up create mode 100755 docker/scripts/start.sh create mode 100644 docker/templates/config.json.tmpl create mode 100644 docker/templates/refstack.conf.tmpl create mode 100755 run-in-docker diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..c6a42acb --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +*.egg* +*.py[cod] +.coverage +.testrepository/ +.tox/ +.venv/ +AUTHORS +ChangeLog +build/ +cover/ +dist +.git/ \ No newline at end of file diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..5afa7a6e --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,45 @@ +FROM ubuntu:14.04 + +RUN \ + groupadd dev && \ + useradd -g dev -s /bin/bash -d /home/dev -m dev && \ + ( umask 226 && echo "dev ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/50_dev ) + +ENV DEBIAN_FRONTEND noninteractive + +RUN \ + apt-get update && \ + apt-get -y install \ + sudo git vim wget \ + nginx \ + python-dev python3-dev python-pip \ + libmysqlclient-dev mysql-client-5.6 mysql-server-5.6 \ + npm && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf /var/lib/mysql/* && \ + rm -rf /etc/nginx/sites-enabled/default + +RUN \ + pip install virtualenv tox ipython ipdb httpie && \ + npm install -g bower && \ + ln -s /usr/bin/nodejs /usr/bin/node + +ADD /docker/scripts/* /usr/bin/ +ADD . /refstack + +ENV PYTHONPATH=/home/dev/refstack \ + SQL_DIR=/home/dev/mysql + +ENV REFSTACK_MYSQL_URL="mysql+pymysql://root@localhost/refstack?unix_socket=${SQL_DIR}/mysql.socket&charset=utf8" + +USER dev + +RUN \ + echo "cd /home/dev/refstack" >> /home/dev/.bashrc &&\ + echo "alias activate='source /home/dev/refstack/.venv/bin/activate'" >> /home/dev/.bashrc &&\ + echo "alias mysql='mysql --no-defaults -S ${SQL_DIR}/mysql.socket'" >> /home/dev/.bashrc &&\ + start.sh &&\ + api-init-db + +CMD start.sh -s +EXPOSE 443 \ No newline at end of file diff --git a/docker/nginx/refstack-site.conf b/docker/nginx/refstack-site.conf new file mode 100644 index 00000000..eafa11ce --- /dev/null +++ b/docker/nginx/refstack-site.conf @@ -0,0 +1,19 @@ +server { + server_name 127.0.0.1; + listen 443 ssl; + + ssl on; + ssl_certificate /etc/nginx/certificates/refstack_dev.crt; + ssl_certificate_key /etc/nginx/certificates/refstack_dev.key; + ssl_protocols TLSv1.1 TLSv1.2; + ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AES:RSA+3DES:!ADH:!AECDH:!MD5:!DSS:!RC4; + ssl_prefer_server_ciphers on; + ssl_session_timeout 5m; + location / { + access_log off; + proxy_pass http://127.0.0.1:8000; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +} \ No newline at end of file diff --git a/docker/nginx/refstack_dev.crt b/docker/nginx/refstack_dev.crt new file mode 100644 index 00000000..24b5ffee --- /dev/null +++ b/docker/nginx/refstack_dev.crt @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICpzCCAhACCQCjqL+hsTaQ6zANBgkqhkiG9w0BAQsFADCBlzELMAkGA1UEBhMC +VUExEDAOBgNVBAgMB0toYXJraXYxEDAOBgNVBAcMB0toYXJraXYxHTAbBgNVBAoM +FE9wZW5zdGFjayBGb3VuZGF0aW9uMRswGQYDVQQDDBJyZWZzdGFjay5sb2NhbC5v +cmcxKDAmBgkqhkiG9w0BCQEWGXNzbHlwdXNoZW5rb0BtaXJhbnRpcy5jb20wHhcN +MTUwNTIxMTYyNDQ1WhcNMjUwNTE4MTYyNDQ1WjCBlzELMAkGA1UEBhMCVUExEDAO +BgNVBAgMB0toYXJraXYxEDAOBgNVBAcMB0toYXJraXYxHTAbBgNVBAoMFE9wZW5z +dGFjayBGb3VuZGF0aW9uMRswGQYDVQQDDBJyZWZzdGFjay5sb2NhbC5vcmcxKDAm +BgkqhkiG9w0BCQEWGXNzbHlwdXNoZW5rb0BtaXJhbnRpcy5jb20wgZ8wDQYJKoZI +hvcNAQEBBQADgY0AMIGJAoGBAMKfF4KpXa+/Ju5SM/oEuEKxffXh6WnA/eG4FQoP +1JMAKKn4wIsn4umKDHebKBDDIT/nlEuDQC9+dour1UxFhba8kJGh5QCmtp+qiWXj +H2f+U0RwBacHgjZoNOqJ+V88PW949IhD91v/lDYmDNtVUHt7BJw7nrnd5MLJAmBe +3S15AgMBAAEwDQYJKoZIhvcNAQELBQADgYEAU0WxG2amQsEv8qq4Vgps4zUTtnec +Vr6KMU06IrvNF0MEODJhasoQFmr2J6dy90abSqPPEdW76cxi1J6wtIEtNvW81elS +9OvdKwL+BoPFo+4G2VvT5Fj8DEl8goyIRGiK7+gpflS4jDRX57DVujgpVd5Omu7L +7F+OgXFZceBNBJw= +-----END CERTIFICATE----- diff --git a/docker/nginx/refstack_dev.key b/docker/nginx/refstack_dev.key new file mode 100644 index 00000000..146475ae --- /dev/null +++ b/docker/nginx/refstack_dev.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICWwIBAAKBgQDCnxeCqV2vvybuUjP6BLhCsX314elpwP3huBUKD9STACip+MCL +J+Lpigx3mygQwyE/55RLg0AvfnaLq9VMRYW2vJCRoeUAprafqoll4x9n/lNEcAWn +B4I2aDTqiflfPD1vePSIQ/db/5Q2JgzbVVB7ewScO5653eTCyQJgXt0teQIDAQAB +AoGAaPWDqGPOsslUJZMPlPaWqOEwHTsIto/uW5z7O8Ht0plzVLdin6mTJn/c2WRD +50ZU2DH8N/1A0FxTcl/pWIjl4wZPDOQ3W8EVcQ30gqV1vunXOi5jDGulCv0nsDXK +YifHxRDehr+ND20IqsQFv+k4PBBTcOMJ+7YpM+DrLubNAkECQQDmOsKF1jumAMP9 +CIkJ8wzXIzAk07w4QXLK1DMoSQVHI0Zz0KjCyJNNbR1w5J7c3QD4KWbIt/PSWuz/ +L+/G6YTjAkEA2Gf7AhFRv1cLg/l/1SwXtVb9MOh7Gf27XuTZeKyV202Fq0y6FhK/ +AQPPQfWQYcsrLkKwegIERtaY34ALLQPu8wJAUBsz4cOH35u2lc0peXfDCPwqXTX6 +8Iv9OAubfTHjDzx74AJDJfsKHc+Qhd5WVDzlgHNPWxl+UbvnaGcyg8BuxwJAXVAA +wPR83leHRKH5yA6aLnxS8prcMenhuFpPl6Q7ffOgdqu/9bKhn6tn3BYp6rEzbmAd +Po7OD0mLY5wPtZpjlwJAShmD4/1gjmV1aYAxQs6gPDDCr5oycn7jyta59gcwKdAv +zO5eKW1jMd+gg4jk3TiuLECdorUoGGbvIxEeP1gGBA== +-----END RSA PRIVATE KEY----- diff --git a/docker/scripts/api-db-version b/docker/scripts/api-db-version new file mode 100755 index 00000000..fbce84dc --- /dev/null +++ b/docker/scripts/api-db-version @@ -0,0 +1,2 @@ +#!/bin/bash +/home/dev/refstack/.venv/bin/python /home/dev/refstack/bin/refstack-manage version 2>/dev/null \ No newline at end of file diff --git a/docker/scripts/api-init-db b/docker/scripts/api-init-db new file mode 100755 index 00000000..edad5421 --- /dev/null +++ b/docker/scripts/api-init-db @@ -0,0 +1,5 @@ +#!/bin/bash +[[ ${DEBUG_MODE} ]] && set -x +mysql --no-defaults -S ${SQL_DIR}/mysql.socket -e 'CREATE DATABASE refstack;' +cd /home/dev/refstack +.venv/bin/python bin/refstack-manage upgrade --revision head \ No newline at end of file diff --git a/docker/scripts/api-sync b/docker/scripts/api-sync new file mode 100755 index 00000000..e4b27658 --- /dev/null +++ b/docker/scripts/api-sync @@ -0,0 +1,5 @@ +#!/bin/bash +[[ ${DEBUG_MODE} ]] && set -x +echo "Syncing project files..." +rsync -avr /refstack /home/dev --exclude-from '/refstack/.gitignore' > /dev/null && \ +echo "Done!" \ No newline at end of file diff --git a/docker/scripts/api-up b/docker/scripts/api-up new file mode 100755 index 00000000..861d557f --- /dev/null +++ b/docker/scripts/api-up @@ -0,0 +1,5 @@ +#!/bin/bash +[[ ${DEBUG_MODE} ]] && set -x +api-sync +cd /home/dev/refstack +.venv/bin/pecan serve refstack/api/config.py > /dev/null diff --git a/docker/scripts/start.sh b/docker/scripts/start.sh new file mode 100755 index 00000000..a98551c1 --- /dev/null +++ b/docker/scripts/start.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +wait_for_line () { + while read line; do + echo "$line" | grep -q "$1" && break + done < "$2" + # Read the fifo for ever otherwise process would block + cat "$2" >/dev/null & +} + +build_tmpl () { + TEMPLATE="$1" + TARGET="$2" + cat ${TEMPLATE} | \ + while read LINE; do + NEWLINE=$(eval echo ${LINE}) + [[ ! -z "$NEWLINE" ]] && echo ${NEWLINE} + done > ${TARGET} +} + +start_mysql () { + # Start MySQL process for tests + [ ! -d ${SQL_DIR} ] && mkdir ${SQL_DIR} + sudo chown dev:dev ${SQL_DIR} + rm -rf ${SQL_DIR}/out && mkfifo ${SQL_DIR}/out + rm -rf ${SQL_DIR}/mysql.socket + # On systems like Fedora here's where mysqld can be found + PATH=$PATH:/usr/libexec + mysqld --no-defaults --datadir=${SQL_DIR} --pid-file=${SQL_DIR}/mysql.pid \ + --socket=${SQL_DIR}/mysql.socket --skip-networking \ + --skip-grant-tables &> ${SQL_DIR}/out & + # Wait for MySQL to start listening to connections + wait_for_line "mysqld: ready for connections." ${SQL_DIR}/out +} + +build_refstack_env () { + api-sync + cd /home/dev/refstack + [ ! -d .venv ] && virtualenv .venv + .venv/bin/pip install -r requirements.txt + #Install some dev tools + .venv/bin/pip install ipython ipdb httpie + cd /home/dev/refstack + bower install --config.interactive=false + + build_tmpl /refstack/docker/templates/config.json.tmpl /home/dev/refstack/refstack-ui/app/config.json + build_tmpl /refstack/docker/templates/refstack.conf.tmpl /home/dev/refstack.conf + sudo cp /home/dev/refstack.conf /etc +} + +start_nginx () { + [ ! -d /etc/nginx/certificates ] && sudo mkdir /etc/nginx/certificates + sudo cp /refstack/docker/nginx/refstack_dev.key /etc/nginx/certificates + sudo cp /refstack/docker/nginx/refstack_dev.crt /etc/nginx/certificates + sudo cp /refstack/docker/nginx/refstack-site.conf /etc/nginx/sites-enabled/ + sudo nginx +} + +while getopts ":s" opt; do + case ${opt} in + s) SLEEP=true;; + esac +done + +[[ ${DEBUG_MODE} ]] && set -x + +touch /tmp/is-not-ready + +start_mysql +start_nginx +build_refstack_env + +rm -rf /tmp/is-not-ready + +if [[ ${SLEEP} ]]; then + set +x + sleep 1024d +fi diff --git a/docker/templates/config.json.tmpl b/docker/templates/config.json.tmpl new file mode 100644 index 00000000..c16d9ad3 --- /dev/null +++ b/docker/templates/config.json.tmpl @@ -0,0 +1 @@ +{\\"refstackApiUrl\\": \\"https://${REFSTACK_HOST:-127.0.0.1}/v1\\"} diff --git a/docker/templates/refstack.conf.tmpl b/docker/templates/refstack.conf.tmpl new file mode 100644 index 00000000..eb0eb9eb --- /dev/null +++ b/docker/templates/refstack.conf.tmpl @@ -0,0 +1,12 @@ +[DEFAULT] +debug = true +verbose = true + +[api] +static_root = /home/dev/refstack/refstack-ui/app +template_path = /home/dev/refstack/refstack-ui/app +app_dev_mode = true +test_results_url = https://${REFSTACK_HOST:-127.0.0.1}/#/results/%s + +[database] +connection = ${REFSTACK_MYSQL_URL} diff --git a/refstack/api/app.py b/refstack/api/app.py index 6dcaa720..b90977e1 100644 --- a/refstack/api/app.py +++ b/refstack/api/app.py @@ -65,7 +65,7 @@ API_OPTS = [ 'contain some details with debug information.' ), cfg.StrOpt('test_results_url', - default='http://refstack.net/output.html?test_id=%s', + default='http://refstack.net/#/results/%s', help='Template for test result url.' ), cfg.StrOpt('github_api_capabilities_url', @@ -187,4 +187,8 @@ def setup_app(config): )] ) + if CONF.api.app_dev_mode: + LOG.debug('\n\n Refstack is served at %s \n\n', + CONF.api.test_results_url.split('/#/')[0]) + return app diff --git a/requirements.txt b/requirements.txt index 96ff43c8..a7609559 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -SQLAlchemy==0.8.3 +SQLAlchemy>=0.8.3 alembic==0.5.0 #gunicorn 19.1.1 has a bug with threading module gunicorn==18 diff --git a/run-in-docker b/run-in-docker new file mode 100755 index 00000000..bc29c7bf --- /dev/null +++ b/run-in-docker @@ -0,0 +1,153 @@ +#!/bin/bash +TAG=$(BRANCH=$(git status -bs| grep "##" | awk '{print $2}'); echo ${BRANCH##*/}) +IMAGE="refstack:${TAG}" +CONTAINER="refstack_${TAG}" +PROJ_DIR=$(git rev-parse --show-toplevel) + +function usage () { +set +x +echo "Usage: $0 [OPTIONS] [COMMAND]" +echo "Build '${IMAGE}' image if it is does not exist." +echo "Run '${CONTAINER}' container and execute COMMAND in it." +echo "Default COMMAND is 'api-up'" +echo "If container '${CONTAINER}' exists (running or stopped) it will be reused." +echo "If you want to get access to your local Refstack not only from localhost, " +echo "please specify public Refstack host:port in env[REFSTACK_HOST]." +echo "You can customize Refstack API config by editing docker/refstack.conf.tmpl." +echo "It is bash template. You can use \${SOME_ENV_VARIABLE} in it." +echo "Default is 127.0.0.1:8000" +echo "" +echo " -r Force delete '${CONTAINER}' container and run it again." +echo " Main usecase for it - updating config from templates" +echo " -b Force delete '${IMAGE}' image and build it again" +echo " Main usecase for it - force build new python/js env" +echo " -i Run container with isolated MySQL data." +echo " By default MySQL data stores in refstack_data_DATA-BASE-REVISON container" +echo " It reuses if such container exists. If you want to drop DB data, just execute" +echo " sudo docker rm refstack_data_DATA-BASE-REVISON" +echo " -d Turn on debug information" +echo " -h Print this usage message" +echo "" +echo "" +echo "Using examples:" +echo "" +echo "Run Refstack API:" +echo "$ ./run-in-docker" +echo "" +echo "Run Refstack API by hands:" +echo "$ ./run-in-docker bash" +echo "$ activate" +echo "$ pecan serve refstack/api/config.py" +echo "" +echo "Open shell in container:" +echo "$ ./run-in-docker bash" +echo "" +echo "Open mysql console in container:" +echo "$ ./run-in-docker bash" +echo "$ mysql" +} + +build_image () { +sudo docker rm -f ${CONTAINER} +PREV_ID=$(sudo docker images refstack | grep ${TAG} | awk '{print $3}') +echo "Try to build ${IMAGE} image" +sudo docker build -t ${IMAGE} -f ${PROJ_DIR}/docker/Dockerfile ${PROJ_DIR} || exit $? +NEW_ID=$(sudo docker images refstack | grep ${TAG} | awk '{print $3}') +if [[ ${PREV_ID} ]] && [[ ! ${PREV_ID} == ${NEW_ID} ]]; then + sudo docker rmi -f ${PREV_ID} && echo "Previous image removed" +fi +} + +wait_ready() { +while true; do + echo "Wait while container is not ready" + sudo docker exec ${CONTAINER} [ ! -e /tmp/is-not-ready ] && \ + echo "Container ${CONTAINER} is running!" && break + sleep 1 +done +} + +run_container (){ +echo "Stop all other refstack containers" +for id in $(sudo docker ps -q); do + NAME=$(sudo docker inspect --format='{{.Name}}' $id) + if [[ ${NAME} == /refstack_* ]] && [[ ! ${NAME} == "/${CONTAINER}" ]]; then + echo "Stopped container ${NAME}" && sudo docker stop $id + fi +done +if [[ $(sudo docker ps -a | grep "${CONTAINER}") ]]; then + echo "Container ${CONTAINER} exists it is reused" + sudo docker start ${CONTAINER} + wait_ready +else + echo "Try to run container ${CONTAINER}" + sudo docker run -d \ + -e REFSTACK_HOST=${REFSTACK_HOST:-127.0.0.1} \ + -e DEBUG_MODE=${DEBUG_MODE} \ + -v ${PROJ_DIR}:/refstack:ro -p 443:443 --name ${CONTAINER} \ + ${IMAGE} start.sh -s + wait_ready + if [[ ! ${ISOLATED_DB} ]]; then + DB_VERSION=$(sudo docker exec -it ${CONTAINER} api-db-version) + DB_CONTAINER=refstack_data_${DB_VERSION::-1} + sudo docker rm -f ${CONTAINER} + if [[ ! $(sudo docker ps -a | grep "${DB_CONTAINER}") ]]; then + sudo docker run -v /home/dev/mysql --name ${DB_CONTAINER} ubuntu /bin/true + echo "Container with mysql data ${DB_CONTAINER} created" + sudo docker run -d \ + -e REFSTACK_HOST=${REFSTACK_HOST:-127.0.0.1} \ + -e DEBUG_MODE=${DEBUG_MODE} \ + -v ${PROJ_DIR}:/refstack:ro --volumes-from ${DB_CONTAINER} -p 443:443 \ + --name ${CONTAINER} ${IMAGE} + wait_ready + sudo docker exec ${CONTAINER} api-init-db + echo "DB init done" + else + sudo docker run -d \ + -e REFSTACK_HOST=${REFSTACK_HOST:-127.0.0.1} \ + -e DEBUG_MODE=${DEBUG_MODE} \ + -v ${PROJ_DIR}:/refstack:ro --volumes-from ${DB_CONTAINER} -p 443:443 \ + --name ${CONTAINER} ${IMAGE} + echo "Container with mysql data ${DB_CONTAINER} attached to ${CONTAINER}" + wait_ready + fi + + + fi +fi +} + +COMMAND="" +while [[ $1 ]] +do + case "$1" in + -h) usage + exit 0;; + -r) echo "Try to remove old ${CONTAINER} container" + sudo docker rm -f ${CONTAINER} + shift;; + -i) echo "Run container with isolated MySQL data." + echo "By default MySQL data stores in refstack_data_[DATA-BASE-REVISON] container" + echo "It reuses if such container exists. If you want to drop DB data, just execute" + echo "sudo docker rm ${DB_CONTAINER}" + ISOLATED_DB=true + shift;; + -b) FORCE_BUILD=true + shift;; + -d) DEBUG_MODE=true + shift;; + *) COMMAND="${COMMAND} $1" + shift;; + esac +done + +[[ ${DEBUG_MODE} ]] && set -x + +#Build proper image if it does not exist of force rebuild fired +if [[ ${FORCE_BUILD} ]] || [[ ! $(sudo docker images refstack | grep ${TAG}) ]]; then + build_image +fi +#Run or start(if it exists) proper container +[[ ! $(sudo docker ps | grep ${CONTAINER}) ]] && run_container + +sudo docker exec -it ${CONTAINER} ${COMMAND:-api-up} \ No newline at end of file