[goal] Migrate testing to ubuntu focal

As per victoria cycle testing runtime and community goal[1]
we need to migrate upstream CI/CD to Ubuntu Focal(20.04).

-Bump the lower constraints for required deps which added python3.8 support
in their later version.

-Changing the way to install and configure Zookeeper.
Installing Zookeeper from official Apache's tarball.
Adding the possiblity to set the specific Zookeeper version.
Minor change in zookeeper logger.

-Use mariadb JDBC for monasca-thresh in devstack, since Drizzle isn't
compatible with MySql Server v8.0.x which is default in Focal

-Python 3.8 doesn't seem to like dictionary keys changing during
iteration.

Fixing RuntimeError: dictionary keys changed during iteration.

Tech. details:
It runs well in py27: 5 iterations
It runs risky in py37: 7 iterations
It is forbbiden in py38: raised RuntimeError
Fixed with list(dic.items()) or tuple(dic.items())

dic = {'1': 'a', '2': 'b', '3': 'c', '4': 'd', '5': 'e'}
for key, value in dic.items():
    print("Key: {0} Value: {1}".format(key,value))
    del dic[key]
    print(dic)
    dic[key] = value
print(dic)

Story: #2007865
Task: #40197
Depends-On: https://review.opendev.org/756859
Change-Id: Ieb4cf38038ffb4d1a152f8ab3b64a14098c7cbb3
This commit is contained in:
Martin Chacon Piza 2020-09-29 11:19:29 +02:00
parent 721d5e8fb5
commit 43922f8223
13 changed files with 89 additions and 98 deletions

View File

@ -1,7 +1,7 @@
openjdk-8-jdk # dist:xenial,bionic
openjdk-8-jre-headless # dist:bionic
maven # dist:xenial,bionic
jq # dist:xenial,bionic
python-dev # dist:xenial,bionic
build-essential # dist:xenial,bionic
mailutils # dist:xenial,bionic
openjdk-8-jdk # dist:xenial,bionic,focal
openjdk-8-jre-headless # dist:bionic,focal
maven # dist:xenial,bionic,focal
jq # dist:xenial,bionic,focal
python-dev # dist:xenial,bionic,focal
build-essential # dist:xenial,bionic,focal
mailutils # dist:xenial,bionic,focal

View File

@ -109,7 +109,7 @@ sporadicMetricNamespaces:
database:
driverClass: %MONASCA_THRESH_DATABASE_ENGINE%
url: "jdbc:drizzle://%DATABASE_HOST%:%DATABASE_PORT%/mon"
url: "jdbc:mariadb://%DATABASE_HOST%:%DATABASE_PORT%/mon"
user: "%DATABASE_USER%"
password: "%DATABASE_PASSWORD%"
properties:

View File

@ -1,36 +0,0 @@
#
# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Modified from http://packages.ubuntu.com/saucy/zookeeperd
NAME=zookeeper
ZOOCFGDIR=/etc/zookeeper/conf
# seems, that log4j requires the log4j.properties file to be in the classpath
CLASSPATH="$ZOOCFGDIR:/usr/share/java/jline.jar:/usr/share/java/log4j-1.2.jar:/usr/share/java/xercesImpl.jar:/usr/share/java/xmlParserAPIs.jar:/usr/share/java/netty.jar:/usr/share/java/slf4j-api.jar:/usr/share/java/slf4j-log4j12.jar:/usr/share/java/zookeeper.jar"
ZOOCFG="$ZOOCFGDIR/zoo.cfg"
ZOO_LOG_DIR=/var/log/zookeeper
USER=$NAME
GROUP=$NAME
PIDDIR=/var/run/$NAME
PIDFILE=$PIDDIR/$NAME.pid
SCRIPTNAME=/etc/init.d/$NAME
JAVA=/usr/bin/java
ZOOMAIN="org.apache.zookeeper.server.quorum.QuorumPeerMain"
ZOO_LOG4J_PROP="INFO,ROLLINGFILE"
JMXLOCALONLY=false
JAVA_OPTS=""

View File

@ -1,5 +1,6 @@
#
# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP
# Copyright 2020 FUJITSU LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,14 +16,12 @@
# limitations under the License.
#
# From http://packages.ubuntu.com/saucy/zookeeperd
# ZooKeeper Logging Configuration
#
# Format is "<default threshold> (, <appender>)+
log4j.rootLogger=${zookeeper.root.logger}
log4j.rootLogger=INFO, CONSOLE, ROLLINGFILE, TRACEFILE
# Example: console appender only
# log4j.rootLogger=INFO, CONSOLE

View File

@ -1 +0,0 @@
0

View File

@ -71,4 +71,4 @@ server.0=127.0.0.1:2888:3888
# Order of the next 2 properties matters.
# autopurge.snapRetainCount must be before autopurge.purgeInterval.
autopurge.snapRetainCount=3
autopurge.purgeInterval=1
autopurge.purgeInterval=1

View File

@ -0,0 +1,20 @@
[Unit]
Description=Zookeeper Daemon
Documentation=http://zookeeper.apache.org
Requires=network.target
After=network.target
[Service]
Type=forking
WorkingDirectory=/opt/zookeeper
User=zookeeper
Group=zookeeper
Environment=ZOO_LOG_DIR=/var/log/zookeeper
ExecStart=/opt/zookeeper/bin/zkServer.sh start /opt/zookeeper/conf/zoo.cfg
ExecStop=/opt/zookeeper/bin/zkServer.sh stop /opt/zookeeper/conf/zoo.cfg
ExecReload=/opt/zookeeper/bin/zkServer.sh restart /opt/zookeeper/conf/zoo.cfg
TimeoutSec=30
Restart=on-failure
[Install]
WantedBy=default.target

View File

@ -1,6 +1,6 @@
#!/bin/bash
# Copyright 2017 FUJITSU LIMITED
# Copyright 2020 FUJITSU LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -14,53 +14,61 @@
# License for the specific language governing permissions and limitations
# under the License.
# call_order:
# - is_zookeeper_enabled
# - install_zookeeper
# - configure_zookeeper
# - clean_zookeeper
_XTRACE_ZOOKEEPER=$(set +o | grep xtrace)
set +o xtrace
# Set up default directories
ZOOKEEPER_DATA_DIR=$DEST/data/zookeeper
ZOOKEEPER_CONF_DIR=/etc/zookeeper
function is_zookeeper_enabled {
is_service_enabled monasca-zookeeper && return 0
return 1
}
function clean_zookeeper {
sudo rm -rf $ZOOKEEPER_DATA_DIR
apt_get -y purge zookeeper
}
function configure_zookeeper {
if is_zookeeper_enabled; then
sudo cp $PLUGIN_FILES/zookeeper/* $ZOOKEEPER_CONF_DIR
sudo sed -i -e 's|.*dataDir.*|dataDir='$ZOOKEEPER_DATA_DIR'|' $ZOOKEEPER_CONF_DIR/zoo.cfg
sudo rm -rf $ZOOKEEPER_DATA_DIR || true
sudo mkdir -p $ZOOKEEPER_DATA_DIR || true
restart_service zookeeper
echo_summary "Cleaning Monasca Zookeeper"
sudo systemctl disable zookeeper
sudo systemctl stop zookeeper
sudo rm -rf /var/log/zookeeper
sudo rm -rf /var/lib/zookeeper
sudo rm -rf /opt/zookeeper-${ZOOKEEPER_VERSION}
sudo rm -rf /opt/zookeeper
sudo rm -rf /etc/systemd/system/zookeeper.service
sudo systemctl daemon-reload
fi
}
function install_zookeeper {
if is_zookeeper_enabled; then
if is_ubuntu; then
install_package zookeeperd
else
die $LINENO "Don't know how to install zookeeper on this platform"
fi
fi
# NOTE(trebskit) it shouldn't really be done here
# but monasca devstack cannot allow it do be done properly
# we'd have to first refactor parts where services are:
# installed, configured and started in single phase
configure_zookeeper
if is_zookeeper_enabled; then
echo_summary "Install Monasca Zookeeper"
local zookeeper_tarball=zookeeper-${ZOOKEEPER_VERSION}.tar.gz
local zookeeper_tarball_url=${APACHE_ARCHIVES}zookeeper/zookeeper-${ZOOKEEPER_VERSION}/${zookeeper_tarball}
local zookeeper_tarball_dest
zookeeper_tarball_dest=`get_extra_file ${zookeeper_tarball_url}`
sudo groupadd --system zookeeper || true
sudo useradd --system -g zookeeper zookeeper || true
sudo tar -xzf ${zookeeper_tarball_dest} -C /opt
sudo ln -sf /opt/zookeeper-${ZOOKEEPER_VERSION} /opt/zookeeper
sudo cp $PLUGIN_FILES/zookeeper/* /opt/zookeeper/conf
sudo chown -R zookeeper:zookeeper /opt/zookeeper/
sudo mkdir /var/log/zookeeper
sudo chown -R zookeeper:zookeeper /var/log/zookeeper
sudo mkdir /var/lib/zookeeper
sudo chown -R zookeeper:zookeeper /var/lib/zookeeper
sudo cp -f "${MONASCA_API_DIR}"/devstack/files/zookeeper/zookeeper.service /etc/systemd/system/zookeeper.service
sudo chown root:root /etc/systemd/system/kafka.service
sudo chmod 644 /etc/systemd/system/zookeeper.service
sudo systemctl daemon-reload
sudo systemctl enable zookeeper
sudo systemctl start zookeeper || sudo systemctl restart zookeeper
fi
}
$_XTRACE_ZOOKEEPER

View File

@ -1021,7 +1021,7 @@ function install_monasca_thresh {
sudo chmod 0640 /etc/monasca/thresh-config.yml
local dbEngine="org.drizzle.jdbc.DrizzleDriver"
local dbEngine="org.mariadb.jdbc.Driver"
local dbPort=3306
if [[ ${MONASCA_DATABASE_USE_ORM} == "True" ]]; then

View File

@ -114,6 +114,7 @@ INFLUXDB_PYTHON_VERSION=${INFLUXDB_PYTHON_VERSION:-1.7.6}
VERTICA_VERSION=${VERTICA_VERSION:-8.0.0-0}
CASSANDRA_VERSION=${CASSANDRA_VERSION:-311x}
ZOOKEEPER_VERSION=${ZOOKEEPER_VERSION:-3.4.13} # 3.4.13 default in Focal; 3.4.10 default in Bionic
# Kafka deb consists of the version of scala plus the version of kafka
BASE_KAFKA_VERSION=${BASE_KAFKA_VERSION:-2.0.1}
SCALA_VERSION=${SCALA_VERSION:-2.12}

View File

@ -6,12 +6,12 @@ Babel==2.3.4
bandit==1.4.0
bashate==0.5.1
cassandra-driver==3.3.0
cffi==1.7.0
cffi==1.14.0
chardet==3.0.4
cliff==2.8.0
cmd2==0.8.0
coverage==4.0
cryptography==2.1
cryptography==2.7
debtcollector==1.2.0
decorator==3.4.0
doc8==0.6.0
@ -28,7 +28,7 @@ futures==3.0.0
gevent==1.2.2
gitdb==0.6.4
GitPython==1.0.1
greenlet==0.4.10
greenlet==0.4.15
httplib2==0.9.1
idna==2.6
imagesize==0.7.1
@ -36,18 +36,18 @@ influxdb==2.9.2
iso8601==0.1.11
Jinja2==2.10
jsonschema==2.6.0
kazoo==2.2
kazoo==2.8.0
keystoneauth1==3.4.0
keystonemiddleware==4.17.0
linecache2==1.0.0
Mako==0.4.0
MarkupSafe==1.0
MarkupSafe==1.1.1
mccabe==0.2.1
monasca-common==2.16.0
monotonic==0.6
mox3==0.20.0
msgpack-python==0.4.0
netaddr==0.7.18
netaddr==0.7.20
netifaces==0.10.4
openstackdocstheme==2.2.1
os-api-ref==1.4.0
@ -56,7 +56,7 @@ os-testr==1.0.0
oslo.concurrency==3.25.0
oslo.config==5.2.0
oslo.context==2.19.2
oslo.db==4.27.0
oslo.db==6.0.0
oslo.i18n==3.15.3
oslo.log==3.36.0
oslo.middleware==3.31.0
@ -76,7 +76,7 @@ pycadf==1.1.0
pycparser==2.18
Pygments==2.2.0
pyinotify==0.9.6
PyMySQL==0.7.6
PyMySQL==0.8.0
pyparsing==2.1.0
pyperclip==1.5.27
python-dateutil==2.5.3
@ -85,7 +85,7 @@ python-keystoneclient==3.8.0
python-mimeparse==1.6.0
python-subunit==1.0.0
pytz==2013.6
PyYAML==3.12
PyYAML==3.13
reno==3.1.0
requests==2.14.2
requestsexceptions==1.2.0
@ -98,8 +98,8 @@ smmap==0.9.0
snowballstemmer==1.2.1
Sphinx==2.0.0
sphinxcontrib-websupport==1.0.1
SQLAlchemy==1.0.10
sqlalchemy-migrate==0.11.0
SQLAlchemy==1.3.0
sqlalchemy-migrate==0.13.0
sqlparse==0.2.2
statsd==3.2.1
stestr==1.0.0
@ -113,6 +113,6 @@ testtools==2.2.0
traceback2==1.4.0
unittest2==1.1.0
urllib3==1.21.1
voluptuous==0.8.9
voluptuous==0.11.1
WebOb==1.7.1
wrapt==1.7.0

View File

@ -1620,7 +1620,7 @@ class TestAlarmDefinition(AlarmTestBase):
result_def = result.json
self.assertEqual(result_def, expected_def)
for key, value in alarm_def.items():
for key, value in list(alarm_def.items()):
del alarm_def[key]
response = self.simulate_request(

View File

@ -1,7 +1,7 @@
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
oslo.db>=4.27.0 # Apache-2.0
oslo.db>=6.0.0 # Apache-2.0
oslo.config>=5.2.0 # Apache-2.0
oslo.context>=2.19.2 # Apache-2.0
oslo.log>=3.36.0 # Apache-2.0
@ -20,8 +20,8 @@ PasteDeploy>=1.5.0 # MIT
pbr!=2.1.0,>=2.0.0 # Apache-2.0
six>=1.10.0 # MIT
pyparsing>=2.1.0 # MIT
voluptuous>=0.8.9 # BSD License
voluptuous>=0.11.1 # BSD License
eventlet!=0.18.3,!=0.20.1,!=0.21.0,!=0.23.0,!=0.25.0,>=0.18.2 # MIT
monasca-common>=2.16.0 # Apache-2.0
SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT
SQLAlchemy>=1.3.0 # MIT
simplejson>=3.8.1 # MIT