diff --git a/devstack/files/debs/monasca-api b/devstack/files/debs/monasca-api index 1af8cf625..c872aa242 100644 --- a/devstack/files/debs/monasca-api +++ b/devstack/files/debs/monasca-api @@ -1,7 +1,8 @@ -openjdk-8-jdk # dist:xenial,bionic,focal -openjdk-8-jre-headless # dist:bionic,focal -maven # dist:xenial,bionic,focal -jq # dist:xenial,bionic,focal +openjdk-8-jdk # dist:xenial,bionic,focal,jammy +openjdk-8-jre-headless # dist:bionic,focal,jammy +maven # dist:xenial,bionic,focal,jammy +jq # dist:xenial,bionic,focal,jammy python-dev # dist:xenial,bionic,focal -build-essential # dist:xenial,bionic,focal -mailutils # dist:xenial,bionic,focal +build-essential # dist:xenial,bionic,focal,jammy +mailutils # dist:xenial,bionic,focal,jammy +python-is-python3 # dist:focal,jammy \ No newline at end of file diff --git a/devstack/lib/monasca-log.sh b/devstack/lib/monasca-log.sh index 9af8ab976..c0496f0a0 100644 --- a/devstack/lib/monasca-log.sh +++ b/devstack/lib/monasca-log.sh @@ -309,7 +309,7 @@ function install_logstash { tar xzf ${logstash_dest} -C $DEST sudo chown -R $STACK_USER $DEST/logstash-${LOGSTASH_VERSION} - ln -sf $DEST/logstash-${LOGSTASH_VERSION} $LOGSTASH_DIR + sudo ln -sf $DEST/logstash-${LOGSTASH_VERSION} $LOGSTASH_DIR sudo mkdir -p $LOGSTASH_DATA_DIR sudo chown $STACK_USER:monasca $LOGSTASH_DATA_DIR @@ -339,7 +339,7 @@ function install_elasticsearch { tar xzf ${es_dest} -C $DEST sudo chown -R $STACK_USER $DEST/elasticsearch-${ELASTICSEARCH_VERSION} - ln -sf $DEST/elasticsearch-${ELASTICSEARCH_VERSION} $ELASTICSEARCH_DIR + sudo ln -sf $DEST/elasticsearch-${ELASTICSEARCH_VERSION} $ELASTICSEARCH_DIR fi } @@ -364,7 +364,7 @@ function configure_elasticsearch { s|%ES_LOG_DIR%|$ELASTICSEARCH_LOG_DIR|g; " -i $ELASTICSEARCH_CFG_DIR/elasticsearch.yml - ln -sf $ELASTICSEARCH_CFG_DIR/elasticsearch.yml $GATE_CONFIGURATION_DIR/elasticsearch.yml + sudo ln -sf $ELASTICSEARCH_CFG_DIR/elasticsearch.yml $GATE_CONFIGURATION_DIR/elasticsearch.yml echo "[Service]" | sudo tee --append /etc/systemd/system/devstack\@elasticsearch.service > /dev/null echo "LimitNOFILE=$LIMIT_NOFILE" | sudo tee --append /etc/systemd/system/devstack\@elasticsearch.service > /dev/null @@ -420,7 +420,7 @@ function install_kibana { local kibana_version_name kibana_version_name=`_get_kibana_version_name` sudo chown -R $STACK_USER $DEST/${kibana_version_name} - ln -sf $DEST/${kibana_version_name} $KIBANA_DIR + sudo ln -sf $DEST/${kibana_version_name} $KIBANA_DIR fi } @@ -443,7 +443,7 @@ function configure_kibana { s|%KEYSTONE_AUTH_URI%|$KEYSTONE_AUTH_URI|g; " -i $KIBANA_CFG_DIR/kibana.yml - ln -sf $KIBANA_CFG_DIR/kibana.yml $GATE_CONFIGURATION_DIR/kibana.yml + sudo ln -sf $KIBANA_CFG_DIR/kibana.yml $GATE_CONFIGURATION_DIR/kibana.yml fi } @@ -522,7 +522,7 @@ function build_kibana_plugin { yarn --cwd $KIBANA_DEV_DIR kbn bootstrap yarn --cwd $plugin_dir build - local get_version_script="import json; obj = json.load(open('$plugin_dir/package.json')); print obj['version']" + local get_version_script="import json; obj = json.load(open('$plugin_dir/package.json')); print(obj['version'])" local monasca_kibana_plugin_version monasca_kibana_plugin_version=$(python -c "$get_version_script") local pkg="$plugin_dir/build/monasca-kibana-plugin-$monasca_kibana_plugin_version.zip" diff --git a/devstack/plugin.sh b/devstack/plugin.sh index a18426702..6ab592da9 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -1154,7 +1154,7 @@ function install_monasca_agent { if is_service_enabled monasca-agent; then echo_summary "Install Monasca monasca_agent" - apt_get install python-yaml libxml2-dev libxslt1-dev + apt_get install python3-yaml libxml2-dev libxslt1-dev MONASCA_AGENT_EXTRAS="kafka_plugin" if is_service_enabled nova && [ "$VIRT_DRIVER" = "libvirt" ]; then @@ -1262,7 +1262,7 @@ function clean_monasca_agent { apt_get purge libxslt1-dev apt_get purge libxml2-dev - apt_get purge python-yaml + apt_get purge python3-yaml fi } @@ -1271,7 +1271,7 @@ function clean_monasca_agent { function install_nodejs { echo_summary "Install Node.js" - curl -sL https://deb.nodesource.com/setup_10.x | sudo bash - + curl -sL https://deb.nodesource.com/setup_18.x | sudo bash - apt_get install nodejs npm config set registry "http://registry.npmjs.org/"; \ diff --git a/doc/requirements.txt b/doc/requirements.txt index 512d6ca93..c35efa85d 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -4,3 +4,4 @@ os-api-ref>=1.4.0 # Apache-2.0 reno>=3.1.0 # Apache-2.0 openstackdocstheme>=2.2.1 # Apache-2.0 SQLAlchemy>=1.3.0 # MIT +oslo.config>=6.8.0 # Apache-2.0 diff --git a/doc/source/contributor/code.rst b/doc/source/contributor/code.rst index f8fa03f65..4248dfafa 100644 --- a/doc/source/contributor/code.rst +++ b/doc/source/contributor/code.rst @@ -14,5 +14,3 @@ Modules .. toctree:: :maxdepth: 2 - - api/autoindex.rst diff --git a/doc/source/index.rst b/doc/source/index.rst index 4b3cfc2d4..0a9a49889 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -113,6 +113,29 @@ Administrating admin/index +Glossary +------------- +.. toctree:: + :maxdepth: 2 + + glossary + +Installation +------------ + +.. toctree:: + :maxdepth: 2 + + install/index + +User +------------ + +.. toctree:: + :maxdepth: 2 + + user/index + Configuration ------------- @@ -124,7 +147,3 @@ Configuration admin/index cli/index configuration/sample -.. only:: html - glossary - install/index - user/index diff --git a/docker/Dockerfile b/docker/Dockerfile index ff7e9ec53..d8b109f86 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,7 +8,7 @@ ARG CONSTRAINTS_BRANCH=master # Extra Python3 dependencies. # gevent is not in upper constrains and v1.3.6 is not working with # older greenlet. -ARG EXTRA_DEPS="gunicorn gevent==1.5.0 python-memcached influxdb" +ARG EXTRA_DEPS="gunicorn gevent>=21.12.0 python-memcached influxdb" # Always start from `monasca-base` image and use specific tag of it. ARG BASE_TAG=master diff --git a/monasca_api/cmd/monasca_db.py b/monasca_api/cmd/monasca_db.py index 1bfa76b8c..880418fbb 100644 --- a/monasca_api/cmd/monasca_db.py +++ b/monasca_api/cmd/monasca_db.py @@ -41,7 +41,7 @@ def do_detect_revision(): fingerprint = Fingerprint(sql_repository.get_engine()) if fingerprint.revision is None: - print(_FP_NOREVISION % fingerprint.sha1) + print(_FP_NOREVISION % fingerprint.sha256) sys.exit(1) else: print(fingerprint.revision) @@ -52,7 +52,7 @@ def do_fingerprint(): if CONF.command.raw: print(fingerprint.schema_raw, end="") else: - print(fingerprint.sha1) + print(fingerprint.sha256) def do_stamp(): @@ -71,7 +71,7 @@ def do_stamp(): else: fp = Fingerprint(engine) if fp.revision is None: - print(_FP_NOREVISION % fp.sha1) + print(_FP_NOREVISION % fp.sha256) sys.exit(1) rev = fp.revision @@ -110,7 +110,7 @@ def do_version(): def add_command_parsers(subparsers): parser = subparsers.add_parser('fingerprint', - help="Compute SHA1 fingerprint of " + help="Compute SHA256 fingerprint of " "current database schema ") parser.add_argument('-r', '--raw', action='store_true', help='Print raw schema dump used for ' diff --git a/monasca_api/common/repositories/influxdb/metrics_repository.py b/monasca_api/common/repositories/influxdb/metrics_repository.py index 151cf7521..b0a87aab9 100644 --- a/monasca_api/common/repositories/influxdb/metrics_repository.py +++ b/monasca_api/common/repositories/influxdb/metrics_repository.py @@ -214,6 +214,20 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository): return query + def _build_select_all_query(self, dimensions, name, tenant_id, + region, start_timestamp, end_timestamp, + no_record_check_dim): + + from_clause = self._build_from_clause(dimensions, name, tenant_id, + region, start_timestamp, + end_timestamp) + if no_record_check_dim is not None: + query = 'select *' + from_clause + " and {} != ''".format(no_record_check_dim) + else: + query = 'select *' + from_clause + + return query + def _build_statistics_query(self, dimensions, name, tenant_id, region, start_timestamp, end_timestamp, statistics, period, offset, group_by, limit): @@ -263,7 +277,7 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository): # replace ' with \' to make query parsable clean_name = name.replace("'", "\\'") if PY3 \ else name.replace("'", "\\'").encode('utf-8') - where_clause += ' from "{}" '.format(clean_name) + where_clause += ' from "{}"'.format(clean_name) # region where_clause += " where _region = '{}'".format(region) @@ -889,9 +903,25 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository): start_timestamp, end_timestamp) result = self.query_tenant_db(query, tenant_id) - json_dim_name_list = self._build_serie_dimension_values( + json_dim_value_list = self._build_serie_dimension_values( result, dimension_name) - return json_dim_name_list + json_dim_value_list_filtered = list() + for serie in result.raw['series']: + for dim_value_dict in json_dim_value_list: + query = self._build_select_all_query( + dimensions={dimension_name: dim_value_dict['dimension_value']}, + name=serie['name'], + tenant_id=tenant_id, + region=region, + start_timestamp=start_timestamp, + end_timestamp=end_timestamp, + no_record_check_dim=None) + result = self.query_tenant_db(query, tenant_id) + if len(result.raw['series']) > 0: + json_dim_value_list_filtered.append(dim_value_dict) + json_dim_value_list_filtered = sorted(json_dim_value_list_filtered, key=lambda + dim_value_dict: dim_value_dict['dimension_value']) + return json_dim_value_list_filtered except Exception as ex: LOG.exception(ex) raise exceptions.RepositoryException(ex) @@ -905,7 +935,26 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository): end_timestamp) result = self.query_tenant_db(query, tenant_id) json_dim_name_list = self._build_serie_dimension_names(result) - return json_dim_name_list + if metric_name is not None: + json_dim_name_list_filtered = list() + for dim_name_dict in json_dim_name_list: + query = self._build_select_all_query( + dimensions=None, + name=metric_name, + tenant_id=tenant_id, + region=region, + start_timestamp=start_timestamp, + end_timestamp=end_timestamp, + no_record_check_dim=dim_name_dict['dimension_name']) + result = self.query_tenant_db(query, tenant_id) + if len(result.raw['series']) > 0: + json_dim_name_list_filtered.append(dim_name_dict) + + json_dim_name_list_filtered = sorted(json_dim_name_list_filtered, key=lambda + dim_name_dict: dim_name_dict['dimension_name']) + return json_dim_name_list_filtered + else: + return json_dim_name_list except Exception as ex: LOG.exception(ex) raise exceptions.RepositoryException(ex) @@ -915,7 +964,7 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository): uri = 'http://{0}:{1}/ping'.format(CONF.influxdb.ip_address, CONF.influxdb.port) try: - resp = requests.head(url=uri) + resp = requests.head(url=uri, timeout=5) except Exception as ex: LOG.exception(str(ex)) return False, str(ex) diff --git a/monasca_api/db/fingerprint.py b/monasca_api/db/fingerprint.py index 6f75e5a56..e868114a6 100644 --- a/monasca_api/db/fingerprint.py +++ b/monasca_api/db/fingerprint.py @@ -21,7 +21,7 @@ from sqlalchemy.orm import sessionmaker LOG = log.getLogger(__name__) -# Map of SHA1 fingerprints to alembic revisions. Note that this is +# Map of SHA256 fingerprints to alembic revisions. Note that this is # used in the pre-alembic case and does not need to be updated if a # new revision is introduced. _REVS = {"43e5913b0272077321ab6f25ffbcda7149b6284b": "00597b5c8325", @@ -43,8 +43,8 @@ class Fingerprint(object): def __init__(self, engine): metadata = self._get_metadata(engine) self.schema_raw = self._get_schema_raw(metadata) - self.sha1 = self._get_schema_sha1(self.schema_raw) - self.revision = self._get_revision(metadata, engine, self.sha1) + self.sha256 = self._get_schema_sha256(self.schema_raw) + self.revision = self._get_revision(metadata, engine, self.sha256) @staticmethod def _get_metadata(engine): @@ -75,18 +75,18 @@ class Fingerprint(object): return "\n".join(schema_strings) @staticmethod - def _get_schema_sha1(schema_raw): - return hashlib.sha1(encodeutils.to_utf8(schema_raw)).hexdigest() + def _get_schema_sha256(schema_raw): + return hashlib.sha256(encodeutils.to_utf8(schema_raw)).hexdigest() @staticmethod - def _get_revision(metadata, engine, sha1): + def _get_revision(metadata, engine, sha256): # Alembic stores the current version in the DB so check that first # and fall back to the lookup table for the pre-alembic case. versions_table = metadata.tables.get('alembic_version') if versions_table is not None: return Fingerprint._lookup_version_from_db(versions_table, engine) - elif sha1: - return Fingerprint._lookup_version_from_table(sha1) + elif sha256: + return Fingerprint._lookup_version_from_table(sha256) @staticmethod def _get_db_session(engine): @@ -102,9 +102,9 @@ class Fingerprint(object): return session.query(versions_table).one()[0] @staticmethod - def _lookup_version_from_table(sha1): - revision = _REVS.get(sha1) + def _lookup_version_from_table(sha256): + revision = _REVS.get(sha256) if not revision: LOG.warning("Fingerprint: {} does not match any revisions." - .format(sha1)) + .format(sha256)) return revision diff --git a/monasca_api/tests/db/test_fingerprint.py b/monasca_api/tests/db/test_fingerprint.py index 4fceb4589..84aadf052 100644 --- a/monasca_api/tests/db/test_fingerprint.py +++ b/monasca_api/tests/db/test_fingerprint.py @@ -35,11 +35,11 @@ class TestFingerprint(base.BaseTestCase): # table of fingerprints. Since we use a dummy schema, we insert a dummy # entry into the lookup table. fingerprint._REVS[ - hashlib.sha1(b'dummy_schema_raw').hexdigest()] = 'dummy_revision' + hashlib.sha256(b'dummy_schema_raw').hexdigest()] = 'dummy_revision' f = fingerprint.Fingerprint('mock_engine') self.assertEqual(f.schema_raw, 'dummy_schema_raw') - self.assertEqual(f.sha1, hashlib.sha1(b'dummy_schema_raw').hexdigest()) + self.assertEqual(f.sha256, hashlib.sha256(b'dummy_schema_raw').hexdigest()) self.assertEqual(f.revision, 'dummy_revision') @mock.patch('monasca_api.db.fingerprint.Fingerprint._get_db_session') @@ -61,5 +61,5 @@ class TestFingerprint(base.BaseTestCase): f = fingerprint.Fingerprint('mock_engine') self.assertEqual(f.schema_raw, 'dummy_schema_raw') - self.assertEqual(f.sha1, hashlib.sha1(b'dummy_schema_raw').hexdigest()) + self.assertEqual(f.sha256, hashlib.sha256(b'dummy_schema_raw').hexdigest()) self.assertEqual(f.revision, 'dummy_revision') diff --git a/monasca_api/tests/test_repositories.py b/monasca_api/tests/test_repositories.py index e7dfafbf3..2b5242e29 100644 --- a/monasca_api/tests/test_repositories.py +++ b/monasca_api/tests/test_repositories.py @@ -221,18 +221,20 @@ class TestRepoMetricsInfluxDB(base.BaseTestCase): self.assertEqual(result, [{u'dimension_value': hostname}]) - query = ('show tag values from "{metric}"' - ' with key = "{column}"' + query = ('select * from "{metric}"' ' where _region = \'{region}\'' - .format(region=region, metric=metric, column=column)) + .format(region=region, metric=metric)) query += ('' if db_per_tenant else ' and _tenant_id = \'{tenant_id}\'' .format(tenant_id=tenant_id)) + query += (' and "{column}" = \'{hostname}\'' + .format(column=column, + hostname=hostname)) query += (' and time >= {start_timestamp}000000u' ' and time < {end_timestamp}000000u' .format(start_timestamp=start_timestamp, end_timestamp=end_timestamp) if timestamp else '') - mock_client.query.assert_called_once_with(query, database=database) + mock_client.query.assert_called_with(query, database=database) def test_list_dimension_values_with_timestamp(self): self.test_list_dimension_values(timestamp=True) @@ -275,18 +277,19 @@ class TestRepoMetricsInfluxDB(base.BaseTestCase): {u'dimension_name': u'service'} ]) - query = ('show tag keys from "{metric}"' - ' where _region = \'{region}\'' - .format(region=region, metric=metric)) - query += ('' if db_per_tenant else ' and _tenant_id = \'{tenant_id}\'' - .format(tenant_id=tenant_id)) - query += (' and time >= {start_timestamp}000000u' - ' and time < {end_timestamp}000000u' - .format(start_timestamp=start_timestamp, - end_timestamp=end_timestamp) - if timestamp else '') + query_last = ('select * from "{metric}"' + ' where _region = \'{region}\'' + .format(region=region, metric=metric)) + query_last += ('' if db_per_tenant else ' and _tenant_id = \'{tenant_id}\'' + .format(tenant_id=tenant_id)) + query_last += (' and time >= {start_timestamp}000000u' + ' and time < {end_timestamp}000000u' + .format(start_timestamp=start_timestamp, + end_timestamp=end_timestamp) + if timestamp else '') + query_last += (' and service != \'\'') - mock_client.query.assert_called_once_with(query, database=database) + mock_client.query.assert_called_with(query_last, database=database) def test_list_dimension_names_with_timestamp(self): self.test_list_dimension_names(timestamp=True) diff --git a/releasenotes/source/yoga.rst b/releasenotes/source/yoga.rst index 7cd5e908a..43cafdea8 100644 --- a/releasenotes/source/yoga.rst +++ b/releasenotes/source/yoga.rst @@ -3,4 +3,4 @@ Yoga Series Release Notes ========================= .. release-notes:: - :branch: stable/yoga + :branch: unmaintained/yoga diff --git a/setup.cfg b/setup.cfg index 076f2c0fd..a9bd51df4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,7 +7,7 @@ description_file = author = OpenStack author_email = openstack-discuss@lists.openstack.org home_page = https://docs.openstack.org/monasca-api/latest/ -python_requires = >=3.6 +python_requires = >=3.8 classifier = Environment :: OpenStack Intended Audience :: Information Technology @@ -17,9 +17,10 @@ classifier = Programming Language :: Python Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 [files] packages = @@ -65,8 +66,3 @@ autodoc_exclude_modules = monasca_tempest_tests.* api_doc_dir = contributor/api -[build_sphinx] -all_files = 1 -build-dir = doc/build -source-dir = doc/source -warning-is-error = 1 diff --git a/test-requirements.txt b/test-requirements.txt index ff03dd92e..ec1a2d7b9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -23,3 +23,4 @@ testtools>=2.2.0 # MIT tempest>=17.1.0 # Apache-2.0 doc8>=0.6.0 # Apache-2.0 +oslo.config>=6.8.0 # Apache-2.0 \ No newline at end of file diff --git a/tox.ini b/tox.ini index 78f32d07a..cf70c0535 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,8 @@ [tox] -envlist = py38,pep8,cover +envlist = py3,pep8,cover minversion = 2.7 skipsdist = True +ignore_basepython_conflict = True [testenv] basepython = python3 @@ -17,9 +18,10 @@ deps = -r{toxinidir}/test-requirements.txt -r{toxinidir}/requirements.txt .[influxdb,cassandra] -whitelist_externals = bash - find +allowlist_externals = bash + find rm + make commands = find . -type f -name "*.pyc" -delete stestr run {posargs} @@ -49,7 +51,7 @@ skip_install = True usedevelop = False commands = # B101(assert_ussed) - API uses asserts because of performance reasons - # B303 - Fingerprint class uses SHA1 to map fingerprints to alembic revisions. + # B303 - Fingerprint class uses SHA256 to map fingerprints to alembic revisions. bandit -r monasca_api -n5 -s B101,B303 -x monasca_api/tests [testenv:bashate] @@ -76,7 +78,7 @@ commands = [testenv:pdf-docs] deps = {[testenv:docs]deps} envdir = {toxworkdir}/docs -whitelist_externals = +allowlist_externals = make rm commands = @@ -107,7 +109,7 @@ description = Builds developer documentation commands = rm -rf doc/build doc/source/contributor/api {[testenv:checkjson]commands} - python setup.py build_sphinx + sphinx-build -W -b html doc/source/ doc/build/html [testenv:checkniceness] skip_install = True