From a6a840551008b4fbd05652faec403b9bfdf6024e Mon Sep 17 00:00:00 2001 From: Denis Makogon Date: Fri, 2 Dec 2016 18:47:58 +0200 Subject: [PATCH] Prepare for project renaming - refactoring with relative imports to support simple renaming - new tox job to build and simply test code inside docker container --- Dockerfile | 2 +- Dockerfile.env.example | 2 +- README.md | 105 ------------------------- laos/api/controllers/apps.py | 10 +-- laos/api/controllers/routes.py | 6 +- laos/api/controllers/runnable.py | 2 +- laos/api/controllers/tasks.py | 4 +- laos/api/middleware/keystone.py | 2 +- laos/common/config.py | 2 +- laos/common/logger.py | 2 +- laos/common/persistence.py | 2 +- laos/models/app.py | 2 +- laos/tests/common/base.py | 2 +- laos/tests/fakes/functions_api.py | 2 +- laos/tests/functional/base.py | 18 ++--- laos/tests/functional/test_apps.py | 4 +- laos/tests/functional/test_routes.py | 4 +- laos/tests/integration/base.py | 13 ++-- laos/tests/integration/test_apps.py | 4 +- laos/tests/integration/test_routes.py | 4 +- requirements.txt | 2 +- scripts/docker_full.sh | 18 +++++ {laos/service => service}/__init__.py | 0 {laos/service => service}/laos_api.py | 0 setup.py | 2 +- testing.md | 107 ++++++++++++++++++++++++++ tox.ini | 11 ++- 27 files changed, 178 insertions(+), 154 deletions(-) create mode 100755 scripts/docker_full.sh rename {laos/service => service}/__init__.py (100%) rename {laos/service => service}/laos_api.py (100%) create mode 100644 testing.md diff --git a/Dockerfile b/Dockerfile index 0f070bb..d5d4503 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,5 +11,5 @@ RUN pip3 install -r /code/requirements.txt RUN python3 /code/setup.py install -ENTRYPOINT ["python3", "/code/laos/service/laos_api.py"] +ENTRYPOINT ["python3", "/code/service/laos_api.py"] EXPOSE 10001 diff --git a/Dockerfile.env.example b/Dockerfile.env.example index 89c1953..9d4cf63 100644 --- a/Dockerfile.env.example +++ b/Dockerfile.env.example @@ -1,6 +1,6 @@ LAOS_HOST=0.0.0.0 LAOS_PORT=10001 -LAOS_DB=mysql://root:ubuntu@192.168.0.120/functions +LAOS_DB=mysql+pymysql://root:ubuntu@192.168.0.120/functions KEYSTONE_ENDPOINT=http://192.168.0.120:5000/v3 FUNCTIONS_URL=http://192.168.0.120:8080/v1 LAOS_LOG_LEVEL=INFO diff --git a/README.md b/README.md index 62e601c..cc3c622 100644 --- a/README.md +++ b/README.md @@ -168,111 +168,6 @@ Once server is launched you can navigate to: to see recent API docs -Testing (general information) ------------------------------ - -In order to run tests you need to install `Tox`: - - $ pip install tox - -Also, you will need a running MySQL instance with the database migrations applied from the previous step. -Tests are dependent on pre-created MySQL database for persistence. -Please set env var - - $ export TEST_DB_URI=mysql://:@:/ - -Testing: PEP8 -------------- - -In order to run `PEP8` style checks run following command: - - $ tox -e pep8 - - -Testing: Functional -------------------- - -In order to run `functional` tests run following command: - - $ tox -e py35-functional - -Pros: - -* lightweight (controllers and DB models testing) -* no OpenStack required -* no IronFunctions required - -Cons: - -* MySQL server required -* OpenStack authentication is not tested -* IronFunctions API stubbed with fake implementation - -Testing: Integration --------------------- - -Integration tests are dependent on following env variables: - -* TEST_DB_URI - similar to functional tests, database endpoint -* FUNCTIONS_API_URL - IronFunctions API URL (default value - `http://localhost:8080/v1`) -* OS_AUTH_URL - OpenStack Identity endpoint -* OS_PROJECT_NAME - OpenStack user-specific project name -* OS_USERNAME - OpenStack user name -* OS_PASSWORD - OpenStack user user password - -To run tests use following command: - - export TEST_DB_URI=mysql://:@:/ - export FUNCTIONS_API_URL=://:/ - export OS_AUTH_URL=://:/ - export OS_PROJECT_NAME= - export OS_USERNAME= - export OS_PASSWORD= - tox -epy35-integration - -Testing: Coverage regression ----------------------------- - -In order to build quality software it is necessary to keep test coverage at its highest point. -So, as part of `Tox` testing new check was added - functional test coverage regression. -In order to run it use following command: - - $ tox -e py35-functional-regression - -3rd party bugs to resolve -------------------------- - -IronFunctions: - -* https://github.com/iron-io/functions/issues/298 -* https://github.com/iron-io/functions/issues/296 -* https://github.com/iron-io/functions/issues/275 -* https://github.com/iron-io/functions/issues/274 - -TODOs ------ - -Swagger doc: - -* Make swagger doc more explicit on HTTP POST/UPDATE body content -* HTTP headers requests - -IronFunctions: - -* Support app deletion in IronFunctions -* Support tasks listing/showing - -Laos: - -* Tests: integration, functional, units -* Better logging coverage - -Python Functions client: - -* Support logging instance passing in [function-python](https://github.com/iron-io/functions_python) -* python-laosclient (ReST API client and CLI tool) -* App writing examples - Contacts -------- diff --git a/laos/api/controllers/apps.py b/laos/api/controllers/apps.py index 0369e6d..85b83f5 100644 --- a/laos/api/controllers/apps.py +++ b/laos/api/controllers/apps.py @@ -17,9 +17,9 @@ from aiohttp import web from aioservice.http import controller from aioservice.http import requests -from laos.api.views import app as app_view -from laos.common import config -from laos.models import app as app_model +from ...common import config +from ...models import app as app_model +from ..views import app as app_view class AppV1Controller(controller.ServiceController): @@ -202,12 +202,12 @@ class AppV1Controller(controller.ServiceController): stored_app = (await app_model.Apps.find_by( project_id=project_id, name=app_name)).pop() - c.logger.info("Updating an app {} for project: {} with data {}" + c.logger.info("Updating app {} for project: {} with data {}" .format(app_name, project_id, str(data))) return web.json_response( data={ "app": app_view.AppView(stored_app, fn_app).view(), - "message": "App successfully update" + "message": "App successfully updated" }, status=200 ) diff --git a/laos/api/controllers/routes.py b/laos/api/controllers/routes.py index f2a3102..825ad17 100644 --- a/laos/api/controllers/routes.py +++ b/laos/api/controllers/routes.py @@ -19,9 +19,9 @@ from aiohttp import web from aioservice.http import controller from aioservice.http import requests -from laos.api.views import app as app_view -from laos.common import config -from laos.models import app as app_model +from ...common import config +from ...models import app as app_model +from ..views import app as app_view class AppRouteV1Controller(controller.ServiceController): diff --git a/laos/api/controllers/runnable.py b/laos/api/controllers/runnable.py index 7a83a20..a183a7d 100644 --- a/laos/api/controllers/runnable.py +++ b/laos/api/controllers/runnable.py @@ -17,7 +17,7 @@ from aiohttp import web from aioservice.http import controller from aioservice.http import requests -from laos.common import config +from ...common import config class RunnableMixin(object): diff --git a/laos/api/controllers/tasks.py b/laos/api/controllers/tasks.py index dc6adda..43603a4 100644 --- a/laos/api/controllers/tasks.py +++ b/laos/api/controllers/tasks.py @@ -14,12 +14,12 @@ from aiohttp import web -# from laos.models import app as app_model +# from ...models import app as app_model from aioservice.http import controller from aioservice.http import requests -# from laos.common import config +# from ...common import config # TODO(denismakogon): disabled until diff --git a/laos/api/middleware/keystone.py b/laos/api/middleware/keystone.py index 0155225..7f772dd 100644 --- a/laos/api/middleware/keystone.py +++ b/laos/api/middleware/keystone.py @@ -18,7 +18,7 @@ from keystoneclient import client from aiohttp import web -from laos.common import config +from ...common import config async def auth_through_token(app: web.Application, handler): diff --git a/laos/common/config.py b/laos/common/config.py index 4e6c700..18fb8ea 100644 --- a/laos/common/config.py +++ b/laos/common/config.py @@ -16,7 +16,7 @@ import aiomysql import asyncio -from laos.common import utils +from . import utils from functionsclient.v1 import client diff --git a/laos/common/logger.py b/laos/common/logger.py index 1a399cb..24ae807 100644 --- a/laos/common/logger.py +++ b/laos/common/logger.py @@ -16,7 +16,7 @@ import datetime import logging import sys -from laos.common import utils +from . import utils def common_logger_setup( diff --git a/laos/common/persistence.py b/laos/common/persistence.py index a26c46b..f0023fd 100644 --- a/laos/common/persistence.py +++ b/laos/common/persistence.py @@ -15,7 +15,7 @@ import datetime import uuid -from laos.common import config +from . import config class BaseDatabaseModel(object): diff --git a/laos/models/app.py b/laos/models/app.py index feea424..fa9aa8e 100644 --- a/laos/models/app.py +++ b/laos/models/app.py @@ -12,7 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. -from laos.common import persistence +from ..common import persistence class Apps(persistence.BaseDatabaseModel): diff --git a/laos/tests/common/base.py b/laos/tests/common/base.py index b3b5608..8a2e9ae 100644 --- a/laos/tests/common/base.py +++ b/laos/tests/common/base.py @@ -16,7 +16,7 @@ import asyncio import datetime import uvloop -from laos.common import logger as log +from ...common import logger as log class LaosTestsBase(object): diff --git a/laos/tests/fakes/functions_api.py b/laos/tests/fakes/functions_api.py index 1192302..84aca3b 100644 --- a/laos/tests/fakes/functions_api.py +++ b/laos/tests/fakes/functions_api.py @@ -120,7 +120,7 @@ class FakeApps(object): async def delete(self, app_name, loop=None): if app_name not in APPS: raise client.FunctionsAPIException( - "App {} not exist.".format(app_name), 404) + "App {} not found.".format(app_name), 404) else: if APP_ROUTES[app_name]: raise client.FunctionsAPIException( diff --git a/laos/tests/functional/base.py b/laos/tests/functional/base.py index f596087..588cf24 100644 --- a/laos/tests/functional/base.py +++ b/laos/tests/functional/base.py @@ -18,17 +18,17 @@ import uuid from aioservice.http import service -from laos.api.controllers import apps -from laos.api.controllers import routes -from laos.api.controllers import runnable -from laos.api.controllers import tasks -from laos.api.middleware import content_type +from ...api.controllers import apps +from ...api.controllers import routes +from ...api.controllers import runnable +from ...api.controllers import tasks +from ...api.middleware import content_type -from laos.common import config +from ...common import config -from laos.tests.common import base -from laos.tests.common import client -from laos.tests.fakes import functions_api +from ..common import base +from ..common import client +from ..fakes import functions_api class LaosFunctionalTestsBase(base.LaosTestsBase, testtools.TestCase): diff --git a/laos/tests/functional/test_apps.py b/laos/tests/functional/test_apps.py index 9035b9d..7c53dc7 100644 --- a/laos/tests/functional/test_apps.py +++ b/laos/tests/functional/test_apps.py @@ -12,8 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. -from laos.tests.common import apps as apps_suite -from laos.tests.functional import base +from ..common import apps as apps_suite +from ..functional import base class TestApps(base.LaosFunctionalTestsBase, apps_suite.AppsTestSuite): diff --git a/laos/tests/functional/test_routes.py b/laos/tests/functional/test_routes.py index 2bef5fb..edbd3d7 100644 --- a/laos/tests/functional/test_routes.py +++ b/laos/tests/functional/test_routes.py @@ -12,8 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. -from laos.tests.common import routes as routes_suite -from laos.tests.functional import base +from ..common import routes as routes_suite +from ..functional import base class TestAppRoutes(base.LaosFunctionalTestsBase, diff --git a/laos/tests/integration/base.py b/laos/tests/integration/base.py index 9c2cf66..23b8fcc 100644 --- a/laos/tests/integration/base.py +++ b/laos/tests/integration/base.py @@ -12,18 +12,19 @@ # License for the specific language governing permissions and limitations # under the License. -import aiohttp import json import os +from urllib import parse + +import aiohttp import testtools -from laos.common import config -from laos.service import laos_api +from ...common import config +from ..common import base +from ..common import client -from laos.tests.common import base -from laos.tests.common import client -from urllib import parse +from service import laos_api class LaosIntegrationTestsBase(base.LaosTestsBase, testtools.TestCase): diff --git a/laos/tests/integration/test_apps.py b/laos/tests/integration/test_apps.py index 0fac0e8..68392b9 100644 --- a/laos/tests/integration/test_apps.py +++ b/laos/tests/integration/test_apps.py @@ -12,8 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. -from laos.tests.common import apps as apps_suite -from laos.tests.integration import base +from ..common import apps as apps_suite +from ..integration import base class TestIntegrationApps(base.LaosIntegrationTestsBase, diff --git a/laos/tests/integration/test_routes.py b/laos/tests/integration/test_routes.py index 21ad473..815e48e 100644 --- a/laos/tests/integration/test_routes.py +++ b/laos/tests/integration/test_routes.py @@ -12,8 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. -from laos.tests.common import routes as routes_suite -from laos.tests.functional import base +from ..common import routes as routes_suite +from ..functional import base class TestIntegrationAppRoutes(base.LaosFunctionalTestsBase, diff --git a/requirements.txt b/requirements.txt index 185b8a3..90afd4a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ # process, which may cause wedges in the gate later. uvloop==0.6.0 # Apache-2.0 -aioservice==0.0.1 # Apache-2.0 +aioservice==0.0.2 # Apache-2.0 aiomysql==0.0.9 # Apache-2.0 alembic==0.8.8 # MIT click==6.6 # Apache-2.0 diff --git a/scripts/docker_full.sh b/scripts/docker_full.sh new file mode 100755 index 0000000..44bc00d --- /dev/null +++ b/scripts/docker_full.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set +x +set +e +set +i + +docker build -t laos-api -f Dockerfile . 2>&1 +docker_h=$(echo ${DOCKER_HOST} | tr "://" " " |awk '{print $2}') +echo -e "Docker IP address ${docker_h}" +echo -e "OpenStack Identity service URL ${OS_AUTH_URL}" +echo -e "IronFunctions URL ${FUNCTIONS_API_URL}" +echo -e "Persistent storage URI ${TEST_DB_URI}" +docker run -d -p ${docker_h}:10002:10001 --env LAOS_HOST=0.0.0.0 --env LAOS_PORT=10001 --env LAOS_DB=${TEST_DB_URI} --env KEYSTONE_ENDPOINT=${OS_AUTH_URL} --env FUNCTIONS_URL=${FUNCTIONS_API_URL} --env LAOS_LOG_LEVEL=INFO laos-api +sleep 2 +docker ps +echo -e "Service running on ${docker_h}:10002" +curl -X GET http://${docker_h}:10002/api/swagger.json | python -mjson.tool +docker stop -t 1 $(docker ps | grep "${docker_h}:10002" | awk '{print $1}') diff --git a/laos/service/__init__.py b/service/__init__.py similarity index 100% rename from laos/service/__init__.py rename to service/__init__.py diff --git a/laos/service/laos_api.py b/service/laos_api.py similarity index 100% rename from laos/service/laos_api.py rename to service/laos_api.py diff --git a/setup.py b/setup.py index 9f716cf..6a67c69 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ setuptools.setup( packages=setuptools.find_packages(), install_requires=[ "uvloop==0.6.0", - "aioservice==0.0.1", + "aioservice==0.0.2", "aiomysql==0.0.9", "alembic==0.8.8", "click==6.6", diff --git a/testing.md b/testing.md new file mode 100644 index 0000000..c56fb40 --- /dev/null +++ b/testing.md @@ -0,0 +1,107 @@ +Testing +------- + +In order to run tests you need to install `Tox`: + + $ pip install tox + +Also, you will need a running MySQL instance with the database migrations applied from the previous step. +Tests are dependent on pre-created MySQL database for persistence. +Please set env var + + $ export TEST_DB_URI=mysql://:@:/ + +PEP8 style checks +----------------- + +In order to run `PEP8` style checks run following command: + + $ tox -e pep8 + + +Functional testing +------------------ + +In order to run `functional` tests run following command: + + $ tox -e py35-functional + +Pros: + +* lightweight (controllers and DB models testing) +* no OpenStack required +* no IronFunctions required + +Cons: + +* MySQL server required +* OpenStack authentication is not tested +* IronFunctions API stubbed with fake implementation + +Integration integrations +------------------------ + +Integration tests are dependent on following env variables: + +* TEST_DB_URI - similar to functional tests, database endpoint +* FUNCTIONS_API_URL - IronFunctions API URL (default value - `http://localhost:8080/v1`) +* OS_AUTH_URL - OpenStack Identity endpoint +* OS_PROJECT_NAME - OpenStack user-specific project name +* OS_USERNAME - OpenStack user name +* OS_PASSWORD - OpenStack user user password + +To run tests use following command: + + export TEST_DB_URI=mysql://:@:/ + export FUNCTIONS_API_URL=://:/ + export OS_AUTH_URL=://:/ + export OS_PROJECT_NAME= + export OS_USERNAME= + export OS_PASSWORD= + tox -epy35-integration + +Testing: Docker-build +--------------------- + +This type of testing allows to ensure if code can be build inside docker container with no problems. +In order to run this check use following commands:: + + export DOCKER_HOST=tcp://:> + export TEST_DB_URI=mysql://:@:/ + export FUNCTIONS_API_URL=://:/ + export OS_AUTH_URL=://:/ + tox -e docker-build + +During this check Tox: + +* builds an image +* deletes all artifacts (Python3.5 image and recently built image) + +Testing Docker-full +------------------- + +This type of testing allows to ensure if code code can be build and run successfully inside docker container with no problems. +In order to run this check use following commands:: + + export DOCKER_HOST=tcp://:> + export TEST_DB_URI=mysql://:@:/ + export FUNCTIONS_API_URL=://:/ + export OS_AUTH_URL=://:/ + tox -e docker-full + +During this check following operations are performed:: + +* build container from source code +* run container with exposed ports +* request Swagger API doc to see if API is responsive +* tear-down running container + + +Coverage regression testing +--------------------------- + +In order to build quality software it is necessary to keep test coverage at its highest point. +So, as part of `Tox` testing new check was added - functional test coverage regression. +In order to run it use following command: + + $ tox -e py35-functional-regression diff --git a/tox.ini b/tox.ini index ef3ef62..1619134 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ # Project LaOS [tox] -envlist = py35-functional,py35-functional-regression,py35-integration,py35-integration-regression,pep8 +envlist = py35-functional,py35-functional-regression,py35-integration,py35-integration-regression,pep8,docker-build minversion = 1.6 skipsdist = True @@ -14,16 +14,16 @@ passenv = OS_PASSWORD OS_USERNAME OS_PROJECT_NAME + DOCKER_HOST setenv = VIRTUAL_ENV={envdir} usedevelop = True install_command = pip install -U {opts} {packages} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = find . -type f -name "*.pyc" -delete - rm -f .testrepository/times.dbm - python setup.py testr --testr-args='{posargs}' whitelist_externals = find rm + docker [testenv:pep8] commands = flake8 @@ -48,7 +48,10 @@ commands = rm -rf doc/html doc/build python setup.py build_sphinx +[testenv:docker-full] +commands = {toxinidir}/scripts/docker_full.sh + [flake8] -ignore = H202,H404,H405,H501 +ignore = H202,H304,H404,H405,H501 show-source = True exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,migrations