From f45bfd913da7eb8b156cfc3a55eedf70745c3d23 Mon Sep 17 00:00:00 2001 From: Pete Vander Giessen Date: Tue, 25 Jun 2019 11:31:42 -0400 Subject: [PATCH] Refactor snap to work with core18. (#73) Refactor snap to work with core18. Giving the snapcraft.yaml a base property helps tremendously with the efficiency of the build process, and I believe that it puts us in a better position to reliably support non Ubuntu distros going forward. This also bases us on long supported bionic libraries, and gives us a nice place to work from as we add Python 3 and Stein support, as well as general polish and fixes. --- .travis.yaml | 16 + DEMO.md | 3 +- patches/nova/nova_log_error.patch | 31 ++ snap-overlay/snap-openstack.yaml | 4 +- snap-overlay/templates/libvirtd.conf.j2 | 468 ++++++++++++++++++++++++ snap-overlay/templates/virtlogd.conf.j2 | 71 ++++ snap-wrappers/rabbitmq/erl | 2 +- snap/hooks/configure | 14 +- snap/hooks/install | 3 - snap/hooks/post-refresh | 7 - snapcraft.yaml | 78 ++-- tests/basic-test.sh | 47 ++- tests/very-basic-test.sh | 30 ++ 13 files changed, 718 insertions(+), 56 deletions(-) create mode 100644 .travis.yaml create mode 100644 patches/nova/nova_log_error.patch create mode 100644 snap-overlay/templates/libvirtd.conf.j2 create mode 100644 snap-overlay/templates/virtlogd.conf.j2 delete mode 100755 snap/hooks/post-refresh create mode 100755 tests/very-basic-test.sh diff --git a/.travis.yaml b/.travis.yaml new file mode 100644 index 0000000..83ee280 --- /dev/null +++ b/.travis.yaml @@ -0,0 +1,16 @@ +dist: bionic +sudo: required +language: python +python: + - "2.7" +before_install: + - sudo apt -qq update + - sudo apt install -y petname +virtualenv: + system_site_packages: true +install: + - sudo snap install multipass --classic --beta + - sudo snap install snapcraft --classic + - snapcraft +script: + - tests/basic_test.sh diff --git a/DEMO.md b/DEMO.md index ad27a13..f01f4f2 100644 --- a/DEMO.md +++ b/DEMO.md @@ -145,7 +145,8 @@ Answer the questions as follows: cloud type: openstack endpoint: http://10.20.20.1:5000/v3 cert path: none - region: userpass + auth type: userpass + region: microstack region endpoint: http://10.20.20.1:5000/v3 add another region?: N diff --git a/patches/nova/nova_log_error.patch b/patches/nova/nova_log_error.patch new file mode 100644 index 0000000..cd3745a --- /dev/null +++ b/patches/nova/nova_log_error.patch @@ -0,0 +1,31 @@ +diff --git a/nova/virt/libvirt/host.py b/nova/virt/libvirt/host.py +index 65ae0ff2f1..b75adf9269 100644 +--- a/lib/python2.7/site-packages/nova/virt/libvirt/host.py ++++ b/lib/python2.7/site-packages/nova/virt/libvirt/host.py +@@ -503,21 +503,25 @@ class Host(object): + libvirt_version = conn.getLibVersion() + if op(libvirt_version, + versionutils.convert_version_to_int(lv_ver)): ++ LOG.error("version check false because lv_ver too low") + return False + + if hv_ver is not None: + hypervisor_version = conn.getVersion() + if op(hypervisor_version, + versionutils.convert_version_to_int(hv_ver)): ++ LOG.error("version check false because hv_ver too low") + return False + + if hv_type is not None: + hypervisor_type = conn.getType() + if hypervisor_type != hv_type: ++ LOG.error("version check false because hv_type too low") + return False + + return True +- except Exception: ++ except Exception as e: ++ LOG.exception("version check false because exception {}".format(e)) + return False + + def has_min_version(self, lv_ver=None, hv_ver=None, hv_type=None): diff --git a/snap-overlay/snap-openstack.yaml b/snap-overlay/snap-openstack.yaml index 73a1414..24b91aa 100644 --- a/snap-overlay/snap-openstack.yaml +++ b/snap-overlay/snap-openstack.yaml @@ -36,6 +36,8 @@ setup: # TODO add local_settings.py horizon-snap.conf.j2: "{snap_common}/etc/horizon/horizon.conf.d/horizon-snap.conf" horizon-nginx.conf.j2: "{snap_common}/etc/nginx/snap/sites-enabled/horizon.conf" + libvirtd.conf.j2: "{snap_common}/libvirt/libvirtd.conf" + virtlogd.conf.j2: "{snap_common}/libvirt/virtlogd.conf" chmod: "{snap_common}/instances": 0755 entry_points: @@ -298,4 +300,4 @@ entry_points: - "{snap_common}/etc/horizon/horizon.conf.d" log-file: "{snap_common}/log/horizon-api.log" templates: - horizon.ini.j2: "{snap_common}/etc/horizon/uwsgi/snap/horizon.ini" + horizon.ini.j2: "{snap_common}/etc/horizon/uwsgi/snap/horizon.ini" diff --git a/snap-overlay/templates/libvirtd.conf.j2 b/snap-overlay/templates/libvirtd.conf.j2 new file mode 100644 index 0000000..72f9280 --- /dev/null +++ b/snap-overlay/templates/libvirtd.conf.j2 @@ -0,0 +1,468 @@ +# Master libvirt daemon configuration file +# +# For further information consult https://libvirt.org/format.html +# +# NOTE: the tests/daemon-conf regression test script requires +# that each "PARAMETER = VALUE" line in this file have the parameter +# name just after a leading "#". + +################################################################# +# +# Network connectivity controls +# + +# Flag listening for secure TLS connections on the public TCP/IP port. +# NB, must pass the --listen flag to the libvirtd process for this to +# have any effect. +# +# It is necessary to setup a CA and issue server certificates before +# using this capability. +# +# This is enabled by default, uncomment this to disable it +#listen_tls = 0 + +# Listen for unencrypted TCP connections on the public TCP/IP port. +# NB, must pass the --listen flag to the libvirtd process for this to +# have any effect. +# +# Using the TCP socket requires SASL authentication by default. Only +# SASL mechanisms which support data encryption are allowed. This is +# DIGEST_MD5 and GSSAPI (Kerberos5) +# +# This is disabled by default, uncomment this to enable it. +#listen_tcp = 1 + + + +# Override the port for accepting secure TLS connections +# This can be a port number, or service name +# +#tls_port = "16514" + +# Override the port for accepting insecure TCP connections +# This can be a port number, or service name +# +#tcp_port = "16509" + + +# Override the default configuration which binds to all network +# interfaces. This can be a numeric IPv4/6 address, or hostname +# +# If the libvirtd service is started in parallel with network +# startup (e.g. with systemd), binding to addresses other than +# the wildcards (0.0.0.0/::) might not be available yet. +# +#listen_addr = "192.168.0.1" + + +# Flag toggling mDNS advertizement of the libvirt service. +# +# Alternatively can disable for all services on a host by +# stopping the Avahi daemon +# +# This is disabled by default, uncomment this to enable it +#mdns_adv = 1 + +# Override the default mDNS advertizement name. This must be +# unique on the immediate broadcast network. +# +# The default is "Virtualization Host HOSTNAME", where HOSTNAME +# is substituted for the short hostname of the machine (without domain) +# +#mdns_name = "Virtualization Host Joe Demo" + + +################################################################# +# +# UNIX socket access controls +# + +# Set the UNIX domain socket group ownership. This can be used to +# allow a 'trusted' set of users access to management capabilities +# without becoming root. +# +# This is restricted to 'root' by default. +unix_sock_group = "sudo" + +# Set the UNIX socket permissions for the R/O socket. This is used +# for monitoring VM status only +# +# Default allows any user. If setting group ownership, you may want to +# restrict this too. +# unix_sock_ro_perms = "0777" + +# Set the UNIX socket permissions for the R/W socket. This is used +# for full management of VMs +# +# Default allows only root. If PolicyKit is enabled on the socket, +# the default will change to allow everyone (eg, 0777) +# +# If not using PolicyKit and setting group ownership for access +# control, then you may want to relax this too. +unix_sock_rw_perms = "0770" + +# Set the UNIX socket permissions for the admin interface socket. +# +# Default allows only owner (root), do not change it unless you are +# sure to whom you are exposing the access to. +#unix_sock_admin_perms = "0700" + +# Set the name of the directory in which sockets will be found/created. +unix_sock_dir = "{{ snap_common }}/run/libvirt" + + + +################################################################# +# +# Authentication. +# +# - none: do not perform auth checks. If you can connect to the +# socket you are allowed. This is suitable if there are +# restrictions on connecting to the socket (eg, UNIX +# socket permissions), or if there is a lower layer in +# the network providing auth (eg, TLS/x509 certificates) +# +# - sasl: use SASL infrastructure. The actual auth scheme is then +# controlled from /etc/sasl2/libvirt.conf. For the TCP +# socket only GSSAPI & DIGEST-MD5 mechanisms will be used. +# For non-TCP or TLS sockets, any scheme is allowed. +# +# - polkit: use PolicyKit to authenticate. This is only suitable +# for use on the UNIX sockets. The default policy will +# require a user to supply their own password to gain +# full read/write access (aka sudo like), while anyone +# is allowed read/only access. +# +# Set an authentication scheme for UNIX read-only sockets +# By default socket permissions allow anyone to connect +# +# To restrict monitoring of domains you may wish to enable +# an authentication mechanism here +auth_unix_ro = "none" + +# Set an authentication scheme for UNIX read-write sockets +# By default socket permissions only allow root. If PolicyKit +# support was compiled into libvirt, the default will be to +# use 'polkit' auth. +# +# If the unix_sock_rw_perms are changed you may wish to enable +# an authentication mechanism here +auth_unix_rw = "none" + +# Change the authentication scheme for TCP sockets. +# +# If you don't enable SASL, then all TCP traffic is cleartext. +# Don't do this outside of a dev/test scenario. For real world +# use, always enable SASL and use the GSSAPI or DIGEST-MD5 +# mechanism in /etc/sasl2/libvirt.conf +#auth_tcp = "sasl" + +# Change the authentication scheme for TLS sockets. +# +# TLS sockets already have encryption provided by the TLS +# layer, and limited authentication is done by certificates +# +# It is possible to make use of any SASL authentication +# mechanism as well, by using 'sasl' for this option +#auth_tls = "none" + + +# Change the API access control scheme +# +# By default an authenticated user is allowed access +# to all APIs. Access drivers can place restrictions +# on this. By default the 'nop' driver is enabled, +# meaning no access control checks are done once a +# client has authenticated with libvirtd +# +#access_drivers = [ "polkit" ] + +################################################################# +# +# TLS x509 certificate configuration +# + + +# Override the default server key file path +# +#key_file = "/etc/pki/libvirt/private/serverkey.pem" + +# Override the default server certificate file path +# +#cert_file = "/etc/pki/libvirt/servercert.pem" + +# Override the default CA certificate path +# +#ca_file = "/etc/pki/CA/cacert.pem" + +# Specify a certificate revocation list. +# +# Defaults to not using a CRL, uncomment to enable it +#crl_file = "/etc/pki/CA/crl.pem" + + + +################################################################# +# +# Authorization controls +# + + +# Flag to disable verification of our own server certificates +# +# When libvirtd starts it performs some sanity checks against +# its own certificates. +# +# Default is to always run sanity checks. Uncommenting this +# will disable sanity checks which is not a good idea +#tls_no_sanity_certificate = 1 + +# Flag to disable verification of client certificates +# +# Client certificate verification is the primary authentication mechanism. +# Any client which does not present a certificate signed by the CA +# will be rejected. +# +# Default is to always verify. Uncommenting this will disable +# verification - make sure an IP whitelist is set +#tls_no_verify_certificate = 1 + + +# A whitelist of allowed x509 Distinguished Names +# This list may contain wildcards such as +# +# "C=GB,ST=London,L=London,O=Red Hat,CN=*" +# +# See the POSIX fnmatch function for the format of the wildcards. +# +# NB If this is an empty list, no client can connect, so comment out +# entirely rather than using empty list to disable these checks +# +# By default, no DN's are checked +#tls_allowed_dn_list = ["DN1", "DN2"] + + +# A whitelist of allowed SASL usernames. The format for username +# depends on the SASL authentication mechanism. Kerberos usernames +# look like username@REALM +# +# This list may contain wildcards such as +# +# "*@EXAMPLE.COM" +# +# See the POSIX fnmatch function for the format of the wildcards. +# +# NB If this is an empty list, no client can connect, so comment out +# entirely rather than using empty list to disable these checks +# +# By default, no Username's are checked +#sasl_allowed_username_list = ["joe@EXAMPLE.COM", "fred@EXAMPLE.COM" ] + + +# Override the compile time default TLS priority string. The +# default is usually "NORMAL" unless overridden at build time. +# Only set this is it is desired for libvirt to deviate from +# the global default settings. +# +#tls_priority="NORMAL" + + +################################################################# +# +# Processing controls +# + +# The maximum number of concurrent client connections to allow +# over all sockets combined. +#max_clients = 5000 + +# The maximum length of queue of connections waiting to be +# accepted by the daemon. Note, that some protocols supporting +# retransmission may obey this so that a later reattempt at +# connection succeeds. +#max_queued_clients = 1000 + +# The maximum length of queue of accepted but not yet +# authenticated clients. The default value is 20. Set this to +# zero to turn this feature off. +#max_anonymous_clients = 20 + +# The minimum limit sets the number of workers to start up +# initially. If the number of active clients exceeds this, +# then more threads are spawned, up to max_workers limit. +# Typically you'd want max_workers to equal maximum number +# of clients allowed +#min_workers = 5 +#max_workers = 20 + + +# The number of priority workers. If all workers from above +# pool are stuck, some calls marked as high priority +# (notably domainDestroy) can be executed in this pool. +#prio_workers = 5 + +# Limit on concurrent requests from a single client +# connection. To avoid one client monopolizing the server +# this should be a small fraction of the global max_workers +# parameter. +#max_client_requests = 5 + +# Same processing controls, but this time for the admin interface. +# For description of each option, be so kind to scroll few lines +# upwards. + +#admin_min_workers = 1 +#admin_max_workers = 5 +#admin_max_clients = 5 +#admin_max_queued_clients = 5 +#admin_max_client_requests = 5 + +################################################################# +# +# Logging controls +# + +# Logging level: 4 errors, 3 warnings, 2 information, 1 debug +# basically 1 will log everything possible +# Note: Journald may employ rate limiting of the messages logged +# and thus lock up the libvirt daemon. To use the debug level with +# journald you have to specify it explicitly in 'log_outputs', otherwise +# only information level messages will be logged. +#log_level = 3 + +# Logging filters: +# A filter allows to select a different logging level for a given category +# of logs +# The format for a filter is one of: +# x:name +# x:+name + +# where name is a string which is matched against the category +# given in the VIR_LOG_INIT() at the top of each libvirt source +# file, e.g., "remote", "qemu", or "util.json" (the name in the +# filter can be a substring of the full category name, in order +# to match multiple similar categories), the optional "+" prefix +# tells libvirt to log stack trace for each message matching +# name, and x is the minimal level where matching messages should +# be logged: + +# 1: DEBUG +# 2: INFO +# 3: WARNING +# 4: ERROR +# +# Multiple filters can be defined in a single @filters, they just need to be +# separated by spaces. +# +# e.g. to only get warning or errors from the remote layer and only errors +# from the event layer: +#log_filters="3:remote 4:event" + +# Logging outputs: +# An output is one of the places to save logging information +# The format for an output can be: +# x:stderr +# output goes to stderr +# x:syslog:name +# use syslog for the output and use the given name as the ident +# x:file:file_path +# output to a file, with the given filepath +# x:journald +# output to journald logging system +# In all case the x prefix is the minimal level, acting as a filter +# 1: DEBUG +# 2: INFO +# 3: WARNING +# 4: ERROR +# +# Multiple outputs can be defined, they just need to be separated by spaces. +# e.g. to log all warnings and errors to syslog under the libvirtd ident: +#log_outputs="3:syslog:libvirtd" +# + +# Log debug buffer size: +# +# This configuration option is no longer used, since the global +# log buffer functionality has been removed. Please configure +# suitable log_outputs/log_filters settings to obtain logs. +#log_buffer_size = 64 + + +################################################################## +# +# Auditing +# +# This setting allows usage of the auditing subsystem to be altered: +# +# audit_level == 0 -> disable all auditing +# audit_level == 1 -> enable auditing, only if enabled on host (default) +# audit_level == 2 -> enable auditing, and exit if disabled on host +# +#audit_level = 2 +# +# If set to 1, then audit messages will also be sent +# via libvirt logging infrastructure. Defaults to 0 +# +#audit_logging = 1 + +################################################################### +# UUID of the host: +# Host UUID is read from one of the sources specified in host_uuid_source. +# +# - 'smbios': fetch the UUID from 'dmidecode -s system-uuid' +# - 'machine-id': fetch the UUID from /etc/machine-id +# +# The host_uuid_source default is 'smbios'. If 'dmidecode' does not provide +# a valid UUID a temporary UUID will be generated. +# +# Another option is to specify host UUID in host_uuid. +# +# Keep the format of the example UUID below. UUID must not have all digits +# be the same. + +# NB This default all-zeros UUID will not work. Replace +# it with the output of the 'uuidgen' command and then +# uncomment this entry +#host_uuid = "00000000-0000-0000-0000-000000000000" +#host_uuid_source = "smbios" + +################################################################### +# Keepalive protocol: +# This allows libvirtd to detect broken client connections or even +# dead clients. A keepalive message is sent to a client after +# keepalive_interval seconds of inactivity to check if the client is +# still responding; keepalive_count is a maximum number of keepalive +# messages that are allowed to be sent to the client without getting +# any response before the connection is considered broken. In other +# words, the connection is automatically closed approximately after +# keepalive_interval * (keepalive_count + 1) seconds since the last +# message received from the client. If keepalive_interval is set to +# -1, libvirtd will never send keepalive requests; however clients +# can still send them and the daemon will send responses. When +# keepalive_count is set to 0, connections will be automatically +# closed after keepalive_interval seconds of inactivity without +# sending any keepalive messages. +# +#keepalive_interval = 5 +#keepalive_count = 5 + +# +# These configuration options are no longer used. There is no way to +# restrict such clients from connecting since they first need to +# connect in order to ask for keepalive. +# +#keepalive_required = 1 +#admin_keepalive_required = 1 + +# Keepalive settings for the admin interface +#admin_keepalive_interval = 5 +#admin_keepalive_count = 5 + +################################################################### +# Open vSwitch: +# This allows to specify a timeout for openvswitch calls made by +# libvirt. The ovs-vsctl utility is used for the configuration and +# its timeout option is set by default to 5 seconds to avoid +# potential infinite waits blocking libvirt. +# +#ovs_timeout = 5 diff --git a/snap-overlay/templates/virtlogd.conf.j2 b/snap-overlay/templates/virtlogd.conf.j2 new file mode 100644 index 0000000..afa9dfe --- /dev/null +++ b/snap-overlay/templates/virtlogd.conf.j2 @@ -0,0 +1,71 @@ +# Master virtlogd daemon configuration file +# + +################################################################# +# +# Logging controls +# + +# Logging level: 4 errors, 3 warnings, 2 information, 1 debug +# basically 1 will log everything possible + +#log_level = 3 +log_level = 1 + +# Logging filters: +# A filter allows to select a different logging level for a given category +# of logs +# The format for a filter is one of: +# x:name +# x:+name +# where name is a string which is matched against source file name, +# e.g., "remote", "qemu", or "util/json", the optional "+" prefix +# tells libvirt to log stack trace for each message matching name, +# and x is the minimal level where matching messages should be logged: +# 1: DEBUG +# 2: INFO +# 3: WARNING +# 4: ERROR +# +# Multiple filter can be defined in a single @filters, they just need to be +# separated by spaces. +# +# e.g. to only get warning or errors from the remote layer and only errors +# from the event layer: +#log_filters="3:remote 4:event" +log_filters="1:qemu 3:remote 4:event 3:util.json 3:rpc" + +# Logging outputs: +# An output is one of the places to save logging information +# The format for an output can be: +# x:stderr +# output goes to stderr +# x:syslog:name +# use syslog for the output and use the given name as the ident +# x:file:file_path +# output to a file, with the given filepath +# x:journald +# ouput to the systemd journal +# In all case the x prefix is the minimal level, acting as a filter +# 1: DEBUG +# 2: INFO +# 3: WARNING +# 4: ERROR +# +# Multiple output can be defined, they just need to be separated by spaces. +# e.g. to log all warnings and errors to syslog under the virtlogd ident: +#log_outputs="3:syslog:virtlogd" +# +log_outputs="1:file:{{ snap_common }}/log/libvirtd.log" + +# The maximum number of concurrent client connections to allow +# over all sockets combined. +#max_clients = 1024 + + +# Maximum file size before rolling over. Defaults to 2 MB +#max_size = 2097152 + +# Maximum number of backup files to keep. Defaults to 3, +# not including the primary active file +#max_backups = 3 diff --git a/snap-wrappers/rabbitmq/erl b/snap-wrappers/rabbitmq/erl index 16c6cdc..5c631e9 100755 --- a/snap-wrappers/rabbitmq/erl +++ b/snap-wrappers/rabbitmq/erl @@ -19,7 +19,7 @@ # %CopyrightEnd% # ROOTDIR=$SNAP/usr/lib/erlang -BINDIR=$ROOTDIR/erts-7.3/bin +BINDIR=$ROOTDIR/erts-9.2/bin EMU=beam PROGNAME=`echo $0 | sed 's/.*\///'` export EMU diff --git a/snap/hooks/configure b/snap/hooks/configure index 3e7f7cb..aee0232 100755 --- a/snap/hooks/configure +++ b/snap/hooks/configure @@ -34,7 +34,9 @@ echo "Configuring RabbitMQ" # Rabbitmq isn't always started when we run this. Wait for it to start. while :; do - grep "Starting broker... completed" ${SNAP_COMMON}/log/rabbitmq/startup_log && break + grep "Starting broker..." ${SNAP_COMMON}/log/rabbitmq/startup_log && \ + grep "completed" ${SNAP_COMMON}/log/rabbitmq/startup_log && \ + break echo "waiting for rabbitmq to start" && sleep 1; done @@ -63,5 +65,15 @@ openstack image show cirros || { --public --container-format=bare --disk-format=qcow2 cirros } +# Restart libvirt and virtlogd to get logging +# TODO: figure out why this doesn't Just Work initially +systemctl restart snap.microstack.*virt* + # Wait for horizon while ! nc -z 10.20.20.1 80; do sleep 0.1; done; + +# Restart Placement API +# Workaround for issue w/ base:core18, where the Placement API throws +# http 500s until it has been restarted. +# TODO: root cause and fix the problem. +systemctl restart snap.microstack.nova-uwsgi.service diff --git a/snap/hooks/install b/snap/hooks/install index e693b4e..15587b8 100755 --- a/snap/hooks/install +++ b/snap/hooks/install @@ -1,9 +1,6 @@ #!/bin/sh set -e -install -D $SNAP/var/snap/microstack/common/libvirt/libvirtd.conf $SNAP_COMMON/libvirt/libvirtd.conf -sed -i 's/unix_sock_group = "libvirtd"/unix_sock_group = "sudo"/' $SNAP_COMMON/libvirt/libvirtd.conf - # MySQL snapshot for speedy install # snapshot is a mysql data dir with # rocky keystone,nova,glance,neutron dbs. diff --git a/snap/hooks/post-refresh b/snap/hooks/post-refresh deleted file mode 100755 index 3221002..0000000 --- a/snap/hooks/post-refresh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -set -e - -if [ ! -f $SNAP_COMMON/libvirt/libvirtd.conf ]; then - install -D $SNAP/var/snap/microstack/common/libvirt/libvirtd.conf $SNAP_COMMON/libvirt/libvirtd.conf - sed -i 's/unix_sock_group = "libvirtd"/unix_sock_group = "sudo"/' $SNAP_COMMON/libvirt/libvirtd.conf -fi diff --git a/snapcraft.yaml b/snapcraft.yaml index 976aa23..e40bfd1 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -1,5 +1,6 @@ name: microstack version: rocky +base: core18 summary: OpenStack on your laptop. description: | Microstack gives you an easy way to develop and test OpenStack @@ -229,6 +230,9 @@ apps: daemon: simple environment: LD_LIBRARY_PATH: $SNAP/lib:$SNAP/lib/$SNAPCRAFT_ARCH_TRIPLET:$SNAP/usr/lib:$SNAP/usr/lib/$SNAPCRAFT_ARCH_TRIPLET:$SNAP/usr/lib/$SNAPCRAFT_ARCH_TRIPLET/pulseaudio + virtlogd: + command: virtlogd + daemon: simple virsh: command: virsh @@ -315,7 +319,8 @@ parts: openstack-projects: plugin: python python-version: python2 - constraints: https://raw.githubusercontent.com/openstack/requirements/stable/rocky/upper-constraints.txt + constraints: + - https://raw.githubusercontent.com/openstack/requirements/stable/rocky/upper-constraints.txt source: http://tarballs.openstack.org/keystone/keystone-stable-rocky.tar.gz python-packages: - libvirt-python @@ -333,6 +338,7 @@ parts: - python-swiftclient build-packages: - gcc + - git - libffi-dev - libssl-dev - libxml2-dev @@ -342,19 +348,17 @@ parts: - conntrack - coreutils - haproxy - override-build: | - # Ensure libvirt is discovered from previous built part - export PKG_CONFIG_PATH=$SNAPCRAFT_STAGE/lib/pkgconfig - snapcraftctl build + - libpython2.7 override-prime: | snapcraftctl prime + # Uncomment the below line for better python-libvirt debugging output. + # patch --batch --forward --strip 1 --input ../project/patches/nova/nova_log_error.patch # Now that everything is built, let's disable user site-packages # as stated in PEP-0370 sed -i usr/lib/python2.7/site.py -e 's/^ENABLE_USER_SITE = None$/ENABLE_USER_SITE = False/' # This is the last step, let's now compile all our pyc files. # Ignore errors due to syntax issues in foobar python 2. ./usr/bin/python2.7 -m compileall . || true - keystone-config: after: [openstack-projects] plugin: dump @@ -448,7 +452,7 @@ parts: openvswitch: # TODO: figure out a better way to fetch the version - source: http://openvswitch.org/releases/openvswitch-2.10.1.tar.gz + source: http://openvswitch.org/releases/openvswitch-2.10.2.tar.gz plugin: autotools build-packages: - libssl-dev @@ -522,15 +526,13 @@ parts: done snapcraftctl build - libxml2: - source: http://xmlsoft.org/sources/libxml2-2.9.4.tar.gz - plugin: autotools - # libvirt/qemu qemu: source: . - source-subdir: qemu-2.5+dfsg + source-subdir: qemu-2.11+dfsg plugin: autotools + after: + - openstack-projects stage-packages: - seabios - ipxe-qemu @@ -540,14 +542,14 @@ parts: - libasound2 - libasyncns0 - libbluetooth3 - - libboost-iostreams1.58.0 - - libboost-random1.58.0 - - libboost-system1.58.0 - - libboost-thread1.58.0 + - libboost-iostreams1.62.0 + - libboost-random1.62.0 + - libboost-system1.62.0 + - libboost-thread1.62.0 - libcaca0 - libfdt1 - libflac8 - - libiscsi2 + - libiscsi7 - libjpeg-turbo8 - libnspr4 - libnss3 @@ -567,6 +569,7 @@ parts: - libxau6 - libxcb1 - libxdmcp6 + - libxen-dev - libxext6 build-packages: - acpica-tools @@ -611,9 +614,12 @@ parts: - --enable-system - --target-list=x86_64-softmmu override-build: | - wget http://archive.ubuntu.com/ubuntu/pool/main/q/qemu/qemu_2.5+dfsg.orig.tar.xz - wget http://archive.ubuntu.com/ubuntu/pool/main/q/qemu/qemu_2.5+dfsg-5ubuntu10.38.debian.tar.xz - wget http://archive.ubuntu.com/ubuntu/pool/main/q/qemu/qemu_2.5+dfsg-5ubuntu10.38.dsc + # Clean up old install + rm -R qemu-2.* || true + # Download sources and patch + wget http://archive.ubuntu.com/ubuntu/pool/main/q/qemu/qemu_2.11+dfsg.orig.tar.xz + wget http://archive.ubuntu.com/ubuntu/pool/main/q/qemu/qemu_2.11+dfsg-1ubuntu7.14.debian.tar.xz + wget http://archive.ubuntu.com/ubuntu/pool/main/q/qemu/qemu_2.11+dfsg-1ubuntu7.14.dsc dpkg-source -x qemu_*.dsc snapcraftctl build organize: @@ -630,7 +636,8 @@ parts: libvirt: source: . - source-subdir: libvirt-1.3.1 + source-subdir: libvirt-4.0.0 + after: [openstack-projects, qemu] plugin: autotools build-packages: - libxml2-dev @@ -648,19 +655,22 @@ parts: - libpciaccess-dev - libnl-3-dev - libnl-route-3-dev + - libxml2-utils - uuid-dev - try: [libnuma-dev] + - python-all + - python-six - wget - dpkg-dev + - xsltproc stage-packages: - - try: [dmidecode] + - dmidecode - dnsmasq - - dnsmasq-utils - - ebtables - libxml2 - libyajl2 - try: [libnuma1] - libcurl3-gnutls + - libpcap0.8 - libpciaccess0 configflags: - --with-qemu @@ -687,12 +697,15 @@ parts: - DNSMASQ=/snap/$SNAPCRAFT_PROJECT_NAME/current/usr/sbin/dnsmasq - DMIDECODE=/snap/$SNAPCRAFT_PROJECT_NAME/current/usr/sbin/dmidecode - OVSVSCTL=/snap/$SNAPCRAFT_PROJECT_NAME/current/bin/ovs-vsctl - - EBTABLES_PATH=/snap/$SNAPCRAFT_PROJECT_NAME/current/sbin/ebtables - IPTABLES_PATH=/snap/$SNAPCRAFT_PROJECT_NAME/current/sbin/iptables + - IP6TABLES_PATH=/snap/$SNAPCRAFT_PROJECT_NAME/current/sbin/ip6tables override-build: | - wget http://archive.ubuntu.com/ubuntu/pool/main/libv/libvirt/libvirt_1.3.1.orig.tar.gz - wget http://archive.ubuntu.com/ubuntu/pool/main/libv/libvirt/libvirt_1.3.1-1ubuntu10.25.debian.tar.xz - wget http://archive.ubuntu.com/ubuntu/pool/main/libv/libvirt/libvirt_1.3.1-1ubuntu10.25.dsc + # Clean up old install + rm -R libvirt-4.0.* || true + # Download sources and patch + wget http://archive.ubuntu.com/ubuntu/pool/main/libv/libvirt/libvirt_4.0.0.orig.tar.xz + wget http://archive.ubuntu.com/ubuntu/pool/main/libv/libvirt/libvirt_4.0.0-1ubuntu8.10.debian.tar.xz + wget http://archive.ubuntu.com/ubuntu/pool/main/libv/libvirt/libvirt_4.0.0-1ubuntu8.10.dsc dpkg-source -x libvirt*.dsc snapcraftctl build organize: @@ -733,13 +746,13 @@ parts: # Memcached Token Caching memcached: plugin: autotools - source: https://memcached.org/files/memcached-1.5.10.tar.gz + source: https://memcached.org/files/memcached-1.5.14.tar.gz build-packages: - libevent-dev - gcc - make stage-packages: - - libevent-2.0-5 + - libevent-2.1-6 override-build: | ./configure --prefix=$SNAPCRAFT_PART_INSTALL make @@ -759,7 +772,7 @@ parts: source: https://www.kernel.org/pub/linux/utils/net/bridge-utils/bridge-utils-1.6.tar.gz plugin: autotools iproute2: - source: https://www.kernel.org/pub/linux/utils/net/iproute2/iproute2-4.9.0.tar.gz + source: https://www.kernel.org/pub/linux/utils/net/iproute2/iproute2-4.20.0.tar.gz plugin: autotools build-packages: - bison @@ -773,7 +786,8 @@ parts: configflags: - --disable-nftables - --prefix=/snap/$SNAPCRAFT_PROJECT_NAME/current - install: | + override-build: | + snapcraftctl build cp --remove-destination $SNAPCRAFT_PART_INSTALL/snap/$SNAPCRAFT_PROJECT_NAME/current/sbin/xtables-multi \ $SNAPCRAFT_PART_INSTALL/snap/$SNAPCRAFT_PROJECT_NAME/current/bin/iptables-xml organize: diff --git a/tests/basic-test.sh b/tests/basic-test.sh index a23f368..9b705c9 100755 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -1,22 +1,37 @@ #!/bin/bash +############################################################################## +# +# This is a "basic" test script for Microstack. It will install the +# microstack snap, spin up a test instance, and verify that the test +# instance is accessible, and can access the Internet. +# +# The multipass snap and the petname debian package must be installed +# on the host system in order to run this test. +# +# The basic test accepts two command line arguments: +# +# -u # First installs a released snap from the named +# # channel, in order to test basic upgrade functionality. +# +# -d # Specifies the distro of the multipass vm. +# +############################################################################## +# Configuration and checks set -ex -# Command line args. -# Pass -u edge/candidate/stable to install -# microstack from the matching channel in the snap store before -# installing the locally built snap. This will help verify that we -# aren't breaking snaps in the wild with a change. UPGRADE_FROM="none" -while getopts u: option +DISTRO=18.04 + +while getopts u:d: option do case "${option}" in u) UPGRADE_FROM=${OPTARG};; + d) DISTRO=${OPTARG};; esac done -# Dependencies. TODO: move these into a testing harness command -v multipass > /dev/null || (echo "Please install multipass."; exit 1); command -v petname > /dev/null || (echo "Please install petname."; exit 1); if [ ! -f microstack_rocky_amd64.snap ]; then @@ -26,7 +41,13 @@ if [ ! -f microstack_rocky_amd64.snap ]; then fi MACHINE=$(petname) -DISTRO=18.04 + +# Setup +echo "++++++++++++++++++++++++++++++++++++++++++++++++++" +echo "++ Starting tests on $MACHINE. ++" +echo "++ Distro: $DISTRO ++" +echo "++ Upgrade from: $UPGRADE_FROM ++" +echo "++++++++++++++++++++++++++++++++++++++++++++++++++" # Launch a machine and copy the snap to it. multipass launch --cpus 2 --mem 16G $DISTRO --name $MACHINE @@ -41,6 +62,11 @@ fi # Install the snap under test multipass exec $MACHINE -- \ sudo snap install --classic --dangerous microstack*.snap +# Comment out the above and uncomment below to install the version of +# the snap from the store. +# TODO: add this as a flag. +#multipass exec $MACHINE -- \ +# sudo snap install --classic --edge microstack # Run microstack.launch multipass exec $MACHINE -- /snap/bin/microstack.launch breakfast @@ -86,9 +112,10 @@ done; # Cleanup unset IP -echo "Completed tests. Tearing down $MACHINE." +echo "++++++++++++++++++++++++++++++++++++++++++++++++++" +echo "++ Completed tests. Tearing down $MACHINE. ++" +echo "++++++++++++++++++++++++++++++++++++++++++++++++++" multipass stop $MACHINE multipass delete $MACHINE multipass purge # This is a little bit rude to do, but we assume that # we can beat up on the test machine a bit. - diff --git a/tests/very-basic-test.sh b/tests/very-basic-test.sh new file mode 100755 index 0000000..089e774 --- /dev/null +++ b/tests/very-basic-test.sh @@ -0,0 +1,30 @@ +#!/bin/bash +############################################################################## +# +# This is a "very basic" test script for Microstack. It will install +# the microstack snap on a vm, and dump you into a shell on the vm for +# troubleshooting. +# +# The multipass snap and the petname debian package must be installed +# on the host system in order to run this test. +# +############################################################################## + +set -ex + +UPGRADE_FROM="none" +DISTRO=18.04 +MACHINE=$(petname) + +# Make a vm +multipass launch --cpus 2 --mem 16G $DISTRO --name $MACHINE + +# Install the snap +multipass copy-files microstack_rocky_amd64.snap $MACHINE: +multipass exec $MACHINE -- \ + sudo snap install --classic --dangerous microstack*.snap + +# Drop the user into a snap shell, as root. +multipass exec $MACHINE -- \ + sudo snap run --shell microstack.launch +