Fuel-stats prepared for OpenStack CI

We have 3 services: analytics, collector, migration.
Migration is frozen and going to be removed. Also
migration requires Elasticsearch server for tests,
thus it can't be tested on OpenStack CI.

Pep8 checked on all services.

Project works only on python2.7, thus
gate-fuel-stats-python26 should be removed.

Change-Id: Ic5b6f15903918b4b0850ce0cdc12543925de0cfa
Closes-Bug: #1500073

MANIFEST.in

@@ -0,0 +1,7 @@
+include *.txt
+include collector/collector/test/logs
+graft analytics/static
+graft collector/collector/api/db/migrations
+recursive-include collector/api/schemas *.json
+prune analytics/static/bower_components
+prune analytics/static/node_modules

analytics/fuel_analytics/api/common/__init__.py

@@ -11,4 +11,3 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
-

analytics/fuel_analytics/test/base.py

@@ -25,6 +25,7 @@ from fuel_analytics.api.log import init_logger
 
 # Configuring app for the test environment
 app.config.from_object('fuel_analytics.api.config.Testing')
+app.config.from_envvar('ANALYTICS_SETTINGS', silent=True)
 init_logger()
 
 

collector/collector/test/base.py

@@ -27,6 +27,7 @@ flask_migrate.Migrate(app, db)
 
 # Configuring app for the test environment
 app.config.from_object('collector.api.config.Testing')
+app.config.from_envvar('COLLECTOR_SETTINGS', silent=True)
 init_logger()
 
 

migration/migration/test/test_es_mapping.py

@@ -1,13 +1,13 @@
 #    Copyright 2014 Mirantis, Inc.
 #
-#    Licensed under the Apache License, Version 2.0 (the 'License'); you may
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
 #    a copy of the License at
 #
 #         http://www.apache.org/licenses/LICENSE-2.0
 #
 #    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an 'AS IS' BASIS, WITHOUT
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.

requirements.txt

@@ -0,0 +1,11 @@
+alembic==0.6.7
+Flask==0.10.1
+Flask-JsonSchema==0.1.1
+Flask-Migrate==1.2.0
+Flask-Script==2.0.5
+Flask-SQLAlchemy==2.0
+psycopg2==2.5.4
+python-dateutil==2.2
+PyYAML==3.11
+six>=1.8.0
+SQLAlchemy==0.9.8

setup.py

@@ -0,0 +1,64 @@
+#    Copyright 2014 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+from setuptools import find_packages
+from setuptools import setup
+
+
+def parse_requirements_txt():
+    root = os.path.dirname(os.path.abspath(__file__))
+    requirements = []
+    with open(os.path.join(root, 'requirements.txt'), 'r') as f:
+        for line in f.readlines():
+            line = line.rstrip()
+            if not line or line.startswith('#'):
+                continue
+            requirements.append(line)
+    return requirements
+
+
+setup(
+    name='fuel-stats',
+    version='0.0.1',
+    description="Service of collecting usage statistics",
+    long_description="""Service of collecting usage statistics""",
+    license="http://www.apache.org/licenses/LICENSE-2.0",
+    classifiers=[
+        "License :: OSI Approved :: Apache Software License",
+        "Development Status :: 3 - Alpha",
+        "Programming Language :: Python",
+        "Topic :: Internet :: WWW/HTTP",
+        "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
+    ],
+    author='Mirantis Inc.',
+    author_email='product@mirantis.com',
+    url='https://mirantis.com',
+    keywords='fuel statistics collector mirantis',
+    package_dir={'collector': 'collector/collector',
+                 'fuel_analytics': 'analytics/fuel_analytics',
+                 'migration': 'migration/migration'},
+    packages=find_packages(where='collector') +
+             find_packages(where='analytics') +
+             find_packages(where='migration'),
+    zip_safe=False,
+    install_requires=parse_requirements_txt(),
+    include_package_data=True,
+    scripts=[
+        'analytics/manage_analytics.py',
+        'collector/manage_collector.py',
+        'migration/manage_migration.py',
+    ]
+)

test-requirements.txt

@@ -0,0 +1,7 @@
+-r requirements.txt
+hacking==0.9.2
+mock==1.0.1
+nose==1.3.4
+nose2==0.4.7
+tox==1.8.0
+unittest2==0.5.1

tools/prepare_ci_config.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+cat > ${FUELSTAT_CI_CONFIG} <<EOL
+SQLALCHEMY_DATABASE_URI = 'postgresql://${FUELSTAT_DB_USER}:${FUELSTAT_DB_PW}@localhost/${FUELSTAT_DB}'
+SQLALCHEMY_ECHO = False
+EOL

tools/prepare_database.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+echo "Preparing pgpass file ${FUELSTAT_DB_ROOTPGPASS}"
+echo "*:*:*:${FUELSTAT_DB_ROOT}:${FUELSTAT_DB_ROOTPW}" > ${FUELSTAT_DB_ROOTPGPASS}
+chmod 600 ${FUELSTAT_DB_ROOTPGPASS}
+
+export PGPASSFILE=${FUELSTAT_DB_ROOTPGPASS}
+
+echo "Trying to find out if role ${FUELSTAT_DB_USER} exists"
+root_roles=$(psql -h 127.0.0.1 -U ${FUELSTAT_DB_ROOT} -t -c "SELECT 'HERE' from pg_roles where rolname='${FUELSTAT_DB_USER}'")
+if [[ ${root_roles} == *HERE ]];then
+  echo "Role ${FUELSTAT_DB_USER} exists. Setting password ${FUELSTAT_DB_PW}"
+  psql -h 127.0.0.1 -U ${FUELSTAT_DB_ROOT} -c "ALTER ROLE ${FUELSTAT_DB_USER} WITH SUPERUSER LOGIN PASSWORD '${FUELSTAT_DB_PW}'"
+else
+  echo "Creating role ${FUELSTAT_DB_USER} with password ${FUELSTAT_DB_PASSWD}"
+  psql -h 127.0.0.1 -U ${FUELSTAT_DB_ROOT} -c "CREATE ROLE ${FUELSTAT_DB_USER} WITH SUPERUSER LOGIN PASSWORD '${FUELSTAT_DB_PW}'"
+fi
+
+echo "Dropping database ${FUELSTAT_DB} if exists"
+psql -h 127.0.0.1 -U ${FUELSTAT_DB_ROOT} -c "DROP DATABASE IF EXISTS ${FUELSTAT_DB}"
+echo "Creating database ${FUELSTAT_DB}"
+psql -h 127.0.0.1 -U ${FUELSTAT_DB_ROOT} -c "CREATE DATABASE ${FUELSTAT_DB} OWNER ${FUELSTAT_DB_USER}"

tox.ini

@@ -0,0 +1,57 @@
+[tox]
+minversion = 1.6
+skipsdist = True
+envlist = py27,pep8
+
+[testenv]
+usedevelop = True
+whitelist_externals = bash
+install_command = pip install {packages}
+setenv = VIRTUAL_ENV={envdir}
+         FUELSTAT_CI_CONFIG={toxinidir}/config_for_ci
+         COLLECTOR_SETTINGS={toxinidir}/config_for_ci
+         ANALYTICS_SETTINGS={toxinidir}/config_for_ci
+         FUELSTAT_DB=openstack_citest
+         FUELSTAT_DB_USER=openstack_citest
+         FUELSTAT_DB_PW=openstack_citest
+         FUELSTAT_DB_ROOT=postgres
+         FUELSTAT_DB_ROOTPW=insecure_slave
+         FUELSTAT_DB_ROOTPGPASS={toxinidir}/pgpass
+deps =
+    -r{toxinidir}/test-requirements.txt
+commands =
+    bash "{toxinidir}/tools/prepare_database.sh"
+    bash "{toxinidir}/tools/prepare_ci_config.sh"
+    python {toxinidir}/collector/manage_collector.py --mode test db upgrade \
+        -d {toxinidir}/collector/collector/api/db/migrations/
+    nosetests {posargs:collector/collector/test analytics/fuel_analytics/test}
+
+[tox:jenkins]
+downloadcache = ~/cache/pip
+
+[testenv:pep8]
+deps = hacking==0.7
+usedevelop = False
+commands =
+    flake8 {posargs:analytics/fuel_analytics collector/collector \
+            migration/migration}
+
+[testenv:cover]
+setenv = NOSE_WITH_COVERAGE=1
+
+[testenv:venv]
+commands = {posargs:}
+
+[testenv:devenv]
+envdir = devenv
+usedevelop = True
+
+[flake8]
+ignore = H302
+exclude = .venv,.git,.tox,dist,doc,*egg,build,docs
+show-pep8 = True
+show-source = True
+count = True
+
+[hacking]
+import_exceptions = testtools.matchers