Switch to ubuntu-minimal for default amphora image

Previously we were using the "ubuntu" diskimage-builder base element as the
default base OS to build the amphora image.
The "ubuntu" element is based on the ubuntu cloud image. This image includes
packages we do not need for the amphora image. At this point it's not clear
that Ubuntu will ship an 18.04 LTS cloud image in the format the "ubuntu"
element requires.
This patch switches the default Ubuntu amphora image to build with the
"ubuntu-minimal" diskimage-builder element.

This patch also moves the amphora agent into a virtual environment inside
the amphora.

It also sets up support for Ubuntu 18.04 (bionic beaver) and HAProxy 1.8.

Change-Id: I84a85ca1363bce2e0f13da64540ec7ba3575e818

devstack/files/debs/octavia

@@ -1 +1,2 @@
 golang
+debootstrap

devstack/files/rpms/octavia

@@ -1 +1,2 @@
 golang
+debootstrap

diskimage-create/diskimage-create.sh

@@ -26,7 +26,7 @@ usage() {
     echo "            [-d **xenial**/**7** | trusty | <other release id> ]"
     echo "            [-e]"
     echo "            [-h]"
-    echo "            [-i **ubuntu** | fedora | centos | rhel ]"
+    echo "            [-i **ubuntu-minimal** | fedora | centos | rhel ]"
     echo "            [-n]"
     echo "            [-o **amphora-x64-haproxy** | <filename> ]"
     echo "            [-p]"
@@ -114,12 +114,16 @@ while getopts "a:b:c:d:ehi:no:pt:r:s:vw:x" opt; do
         i)
             AMP_BASEOS=$OPTARG
             if [ $AMP_BASEOS != "ubuntu" ] && \
+                [ $AMP_BASEOS != "ubuntu-minimal" ] && \
                 [ $AMP_BASEOS != "fedora" ] && \
                 [ $AMP_BASEOS != "centos" ] && \
                 [ $AMP_BASEOS != "rhel" ]; then
                 echo "Error: Unsupported base OS " $AMP_BASEOS " specified"
                 exit 3
             fi
+            if [ $AMP_BASEOS == "ubuntu" ]; then
+                AMP_BASEOS="ubuntu-minimal"
+            fi
         ;;
         n)
             AMP_DISABLE_SSHD=1
@@ -175,9 +179,9 @@ AMP_BACKEND=${AMP_BACKEND:-"haproxy-octavia"}
 
 AMP_CACHEDIR=${AMP_CACHEDIR:-"$HOME/.cache/image-create"}
 
-AMP_BASEOS=${AMP_BASEOS:-"ubuntu"}
+AMP_BASEOS=${AMP_BASEOS:-"ubuntu-minimal"}
 
-if [ "$AMP_BASEOS" = "ubuntu" ]; then
+if [ "$AMP_BASEOS" = "ubuntu-minimal" ]; then
     export DIB_RELEASE=${AMP_DIB_RELEASE:-"xenial"}
 elif [ "${AMP_BASEOS}" = "centos" ] || [ "${AMP_BASEOS}" = "rhel" ]; then
     export DIB_RELEASE=${AMP_DIB_RELEASE:-"7"}
@@ -274,7 +278,7 @@ if [ "$platform" = 'NAME="Ubuntu"' ]; then
 
     # Also check if we can build the BASEOS on this Ubuntu version
     UBUNTU_VERSION=`lsb_release -r | awk '{print $2}'`
-    if [ "$AMP_BASEOS" != "ubuntu" ] && \
+    if [ "$AMP_BASEOS" != "ubuntu-minimal" ] && \
         [ 1 -eq $(echo "$UBUNTU_VERSION < 14.04" | bc) ]; then
             echo "Ubuntu minimum version 14.04 required to build $AMP_BASEOS."
             echo "Earlier versions don't support the extended attributes required."
@@ -371,7 +375,7 @@ fi
 
 # Build the image
 
-if [ "$AMP_BASEOS" = "ubuntu" ]; then
+if [ "$AMP_BASEOS" = "ubuntu-minimal" ]; then
     export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES
 fi
 

elements/amphora-agent/install.d/amphora-agent-source-install/75-amphora-agent-install

@@ -7,11 +7,17 @@ set -eu
 set -o pipefail
 
 SCRIPTDIR=$(dirname $0)
+AMP_VENV=/opt/amphora-agent-venv
 
-pip install -U -c /opt/upper-constraints.txt /opt/amphora-agent
+# Create a virutal environment to contain the amphora agent
+${DIB_PYTHON} -m virtualenv $AMP_VENV
 
-# Accommodate centos default install location
+$AMP_VENV/bin/pip install pip --upgrade
-ln -s /bin/amphora-agent /usr/local/bin/amphora-agent || true
+
+$AMP_VENV/bin/pip install -U -c /opt/upper-constraints.txt /opt/amphora-agent
+
+# Link the amphora-agent out to /usr/local/bin where the startup scripts look
+ln -s $AMP_VENV/bin/amphora-agent /usr/local/bin/amphora-agent || true
 
 mkdir /etc/octavia
 # we assume certs, etc will come in through the config drive

elements/amphora-agent/package-installs.yaml

@@ -8,3 +8,45 @@ libssl-dev:
   installtype: source
 python-dev:
   installtype: source
+
+acl:
+acpid:
+apparmor:
+apt-transport-https:
+at:
+bash-completion:
+cloud-guest-utils:
+cloud-init:
+cron:
+curl:
+dbus:
+dkms:
+dmeventd:
+ethtool:
+gawk:
+ifenslave:
+ifupdown:
+iptables:
+iputils-tracepath:
+irqbalance:
+isc-dhcp-client:
+less:
+logrotate:
+lsof:
+net-tools:
+netbase:
+netcat-openbsd:
+open-vm-tools:
+openssh-client:
+openssh-server:
+pollinate:
+psmisc:
+rsyslog:
+screen:
+socat:
+tcpdump:
+ubuntu-cloudimage-keyring:
+ureadahead:
+uuid-runtime:
+vim-tiny:
+vlan:

elements/haproxy-octavia/package-installs.json

@@ -1,3 +1,4 @@
 {
-    "haproxy": null
+    "haproxy": null,
+    "iputils-ping": null
 }

elements/haproxy-octavia/pkg-map

@@ -14,6 +14,9 @@
     "family": {
         "debian": {
             "haproxy": "haproxy"
+        },
+        "redhat": {
+            "iputils-ping": "iputils"
         }
     },
     "default": {

elements/rebind-sshd/finalise.d/98-rebind-sshd-after-dhcp

@@ -3,6 +3,7 @@
 # isc dhcpd specific section
 if [[ $DISTRO_NAME = "ubuntu" || $DISTRO_NAME = "debian" ]]; then
 
+  mkdir -p /etc/dhcp/dhclient-enter-hooks.d
   echo '#!/bin/sh
 if [ "$reason" = "BOUND" ]; then
     if `grep -q "#ListenAddress 0.0.0.0" /etc/ssh/sshd_config`; then

octavia/amphorae/backends/agent/api_server/listener.py

@@ -201,6 +201,7 @@ class Listener(object):
                             HasIFUPAll=self._osutils.has_ifup_all())
                     text_file.write(text)
 
+        hap_major, hap_minor = haproxy_compatibility.get_haproxy_versions()
         if not os.path.exists(init_path):
             with os.fdopen(os.open(init_path, flags, mode), 'w') as text_file:
 
@@ -215,7 +216,9 @@ class Listener(object):
                                       respawn_interval),
                     amphora_netns=AMPHORA_NETNS,
                     amphora_nsname=consts.AMPHORA_NAMESPACE,
-                    HasIFUPAll=self._osutils.has_ifup_all()
+                    HasIFUPAll=self._osutils.has_ifup_all(),
+                    haproxy_major_version=hap_major,
+                    haproxy_minor_version=hap_minor
                 )
                 text_file.write(text)
 

octavia/amphorae/backends/agent/api_server/templates/systemd.conf.j2

@@ -1,7 +1,5 @@
 [Unit]
 Description=HAProxy Load Balancer
-Documentation=man:haproxy(1)
-Documentation=file:/usr/share/doc/haproxy/configuration.txt.gz
 After=network.target syslog.service {{ amphora_netns }}.service
 Before=octavia-keepalived.service
 Wants=syslog.service
@@ -10,11 +8,26 @@ Requires={{ amphora_netns }}.service
 [Service]
 # Force context as we start haproxy under "ip netns exec"
 SELinuxContext=system_u:system_r:haproxy_t:s0
-EnvironmentFile=-/etc/default/haproxy
+
-ExecStartPre=/usr/sbin/haproxy -f {{ haproxy_cfg }} -f {{ haproxy_user_group_cfg }} -c -q
+Environment="CONFIG={{ haproxy_cfg }}" "USERCONFIG={{ haproxy_user_group_cfg }}" "PIDFILE={{ haproxy_pid }}"
-ExecStart=/sbin/ip netns exec {{ amphora_nsname }} /usr/sbin/haproxy-systemd-wrapper -f {{ haproxy_cfg }} -f {{ haproxy_user_group_cfg }} -p {{ haproxy_pid }} -L {{ peer_name }} $EXTRAOPTS
+
-ExecReload=/usr/sbin/haproxy -c -f {{ haproxy_cfg }} -f {{ haproxy_user_group_cfg }} -L {{ peer_name }}
+ExecStartPre={{ haproxy_cmd }} -f $CONFIG -f $USERCONFIG -c -q
+
+ExecReload={{ haproxy_cmd }} -c -f $CONFIG -f $USERCONFIG -L {{ peer_name }}
 ExecReload=/bin/kill -USR2 $MAINPID
+
+{%- if haproxy_major_version < 2 and haproxy_minor_version < 8 %}
+
+ExecStart=/sbin/ip netns exec {{ amphora_nsname }} {{ haproxy_cmd }}-systemd-wrapper -f $CONFIG -f $USERCONFIG -p $PIDFILE -L {{ peer_name }}
+
+{%- else %}
+
+ExecStart=/sbin/ip netns exec {{ amphora_nsname }} {{ haproxy_cmd }} -Ws -f $CONFIG -f $USERCONFIG -p $PIDFILE -L {{ peer_name }}
+
+Type=notify
+
+{%- endif %}
+
 KillMode=mixed
 Restart=always
 

zuul.d/jobs.yaml

@@ -51,6 +51,14 @@
     run: playbooks/legacy/octavia-v1-dsvm-py3x-scenario/run.yaml
     post-run: playbooks/legacy/octavia-v1-dsvm-py3x-scenario/post.yaml
 
+- job:
+    name: octavia-v1-dsvm-scenario-ubuntu.bionic
+    parent: octavia-v1-dsvm-scenario
+    vars:
+      devstack_localrc:
+        OCTAVIA_AMP_BASE_OS: ubuntu
+        OCTAVIA_AMP_DISTRIBUTION_RELEASE_ID: bionic
+
 - job:
     name: octavia-v1-dsvm-scenario-multinode
     parent: octavia-legacy-dsvm-base-multinode

zuul.d/projects.yaml

@@ -37,6 +37,8 @@
             voting: false
         - octavia-v1-dsvm-scenario-kvm-centos.7:
             voting: false
+        - octavia-v1-dsvm-scenario-ubuntu.bionic:
+            voting: false
     gate:
       queue: octavia
       jobs: