Update to build using charmcraft

Due to a build problem with the reactive plugin, this change falls back
on overriding the steps and doing a manual build, but it also ensures
the CI system builds the charm using charmcraft.  Changes:

- add a build-requirements.txt
- modify charmcraft.yaml
- modify osci.yaml
    -> indicate build with charmcraft
- modify tox.ini
    -> tox -e build does charmcraft build/rename
    -> tox -e build-reactive does the reactive build
- modify bundles to use the <charm>.charm artifact in tests.
  and fix deprecation warning re: prefix
- tox inception to enable tox -e func-test in the CI

Change-Id: Iafe802bd1b01c0447dd45ed697c0ea3d003218e5

build-requirements.txt

@@ -0,0 +1,7 @@
+# NOTES(lourot):
+# * We don't install charmcraft via pip anymore because it anyway spins up a
+#   container and scp the system's charmcraft snap inside it. So the charmcraft
+#   snap is necessary on the system anyway.
+# * `tox -e build` successfully validated with charmcraft 1.2.1
+
+cffi==1.14.6; python_version < '3.6'  # cffi 1.15.0 drops support for py35.

charmcraft.yaml

@@ -2,9 +2,21 @@ type: charm
 
 parts:
   charm:
-    source: src/
+    build-packages:
-    plugin: reactive
+      - tox
-    build-snaps: [charm]
+      - git
+      - python3-dev
+    override-build: |
+      apt-get install ca-certificates -y
+      tox -e build-reactive
+    override-stage: |
+      echo "Copying charm to staging area: $CHARMCRAFT_STAGE"
+      NAME=$(ls $CHARMCRAFT_PART_BUILD/build/builds)
+      cp -r $CHARMCRAFT_PART_BUILD/build/builds/$NAME/* $CHARMCRAFT_STAGE/
+    override-prime: |
+      # For some reason, the normal priming chokes on the fact that there's a
+      # hooks directory.
+      cp -r $CHARMCRAFT_STAGE/* .
 
 bases:
   - name: ubuntu

osci.yaml

@@ -7,3 +7,4 @@
     vars:
       needs_charm_build: true
       charm_build_name: neutron-api-plugin-ironic
+      build_type: charmcraft

rename.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+charm=$(grep "charm_build_name" osci.yaml | awk '{print $2}')
+echo "renaming ${charm}_*.charm to ${charm}.charm"
+echo -n "pwd: "
+pwd
+ls -al
+echo "Removing bad downloaded charm maybe?"
+if [[ -e "${charm}.charm" ]];
+then
+    rm "${charm}.charm"
+fi
+echo "Renaming charm here."
+mv ${charm}_*.charm ${charm}.charm

src/test-requirements.txt

@@ -4,6 +4,9 @@
 #     https://github.com/openstack-charmers/release-tools
 #
 
+# Need tox to be available from tox... inception yes, but its a workaround for now
+tox
+
 # Functional Test Requirements (let Zaza's dependencies solve all dependencies here!)
 git+https://github.com/openstack-charmers/zaza.git#egg=zaza
 git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack

src/tests/bundles/focal-xena.yaml

@@ -177,7 +177,7 @@ services:
       worker-multiplier: 0.25
     storage:
       block-devices: cinder,50G
-    channel: latest/edge
+    channel: yoga/edge
   ceph-radosgw:
     charm: ch:ceph-radosgw
     num_units: 1
@@ -185,7 +185,7 @@ services:
     options:
       source: *source
       namespace-tenants: True
-    channel: latest/edge
+    channel: quincy/edge
   ceph-mon:
     charm: ch:ceph-mon
     num_units: 3
@@ -193,7 +193,7 @@ services:
     options:
       expected-osd-count: 3
       source: *source
-    channel: latest/edge
+    channel: quincy/edge
   ceph-osd:
     charm: ch:ceph-osd
     num_units: 3
@@ -202,7 +202,7 @@ services:
       source: *source
     storage:
       osd-devices: 'cinder,30G'
-    channel: latest/edge
+    channel: quincy/edge
   glance:
     charm: ch:glance
     num_units: 1
@@ -210,7 +210,7 @@ services:
     options:
       openstack-origin: *source
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   keystone:
     charm: ch:keystone
     num_units: 1
@@ -218,7 +218,7 @@ services:
     options:
       openstack-origin: *source
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   neutron-api:
     charm: ch:neutron-api
     num_units: 1
@@ -229,7 +229,7 @@ services:
       openstack-origin: *source
       manage-neutron-plugin-legacy-mode: false
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   neutron-gateway:
     charm: ch:neutron-gateway
     num_units: 1
@@ -239,7 +239,7 @@ services:
       enable-isolated-metadata: true
       enable-metadata-network: true
       bridge-mappings: physnet1:br-ex
-    channel: latest/edge
+    channel: yoga/edge
   nova-cloud-controller:
     charm: ch:nova-cloud-controller
     num_units: 1
@@ -248,7 +248,7 @@ services:
       network-manager: Neutron
       openstack-origin: *source
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   nova-ironic:
     charm: ch:nova-compute
     num_units: 1
@@ -258,7 +258,7 @@ services:
       enable-resize: false
       openstack-origin: *source
       virt-type: ironic
-    channel: latest/edge
+    channel: yoga/edge
   placement:
     charm: ch:placement
     num_units: 1
@@ -266,7 +266,7 @@ services:
     options:
       openstack-origin: *source
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   rabbitmq-server:
     charm: ch:rabbitmq-server
     num_units: 1
@@ -282,7 +282,7 @@ services:
     constraints: mem=2G
     options:
       openstack-origin: *source
-    channel: latest/edge
+    channel: yoga/edge
   ironic-conductor:
     charm: ch:ironic-conductor
     num_units: 1
@@ -293,15 +293,15 @@ services:
       disable-secure-erase: true
       use-ipxe: true
       enabled-network-interfaces: "flat, noop"
-    channel: latest/edge
+    channel: yoga/edge
   neutron-openvswitch:
     charm: ch:neutron-openvswitch
     num_units: 0
     options:
       bridge-mappings: physnet1:br-ex
-    channel: latest/edge
+    channel: yoga/edge
   neutron-ironic-agent:
-    charm: ../../../neutron-api-plugin-ironic
+    charm: ../../../neutron-api-plugin-ironic.charm
     num_units: 0
     options:
       openstack-origin: *source

src/tests/bundles/focal-yoga.yaml

@@ -177,7 +177,7 @@ services:
       worker-multiplier: 0.25
     storage:
       block-devices: cinder,50G
-    channel: latest/edge
+    channel: yoga/edge
   ceph-radosgw:
     charm: ch:ceph-radosgw
     num_units: 1
@@ -185,7 +185,7 @@ services:
     options:
       source: *source
       namespace-tenants: True
-    channel: latest/edge
+    channel: quincy/edge
   ceph-mon:
     charm: ch:ceph-mon
     num_units: 3
@@ -193,7 +193,7 @@ services:
     options:
       expected-osd-count: 3
       source: *source
-    channel: latest/edge
+    channel: quincy/edge
   ceph-osd:
     charm: ch:ceph-osd
     num_units: 3
@@ -202,7 +202,7 @@ services:
       source: *source
     storage:
       osd-devices: 'cinder,30G'
-    channel: latest/edge
+    channel: quincy/edge
   glance:
     charm: ch:glance
     num_units: 1
@@ -210,7 +210,7 @@ services:
     options:
       openstack-origin: *source
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   keystone:
     charm: ch:keystone
     num_units: 1
@@ -218,7 +218,7 @@ services:
     options:
       openstack-origin: *source
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   neutron-api:
     charm: ch:neutron-api
     num_units: 1
@@ -229,7 +229,7 @@ services:
       openstack-origin: *source
       manage-neutron-plugin-legacy-mode: false
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   neutron-gateway:
     charm: ch:neutron-gateway
     num_units: 1
@@ -239,7 +239,7 @@ services:
       enable-isolated-metadata: true
       enable-metadata-network: true
       bridge-mappings: physnet1:br-ex
-    channel: latest/edge
+    channel: yoga/edge
   nova-cloud-controller:
     charm: ch:nova-cloud-controller
     num_units: 1
@@ -248,7 +248,7 @@ services:
       network-manager: Neutron
       openstack-origin: *source
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   nova-ironic:
     charm: ch:nova-compute
     num_units: 1
@@ -258,7 +258,7 @@ services:
       enable-resize: false
       openstack-origin: *source
       virt-type: ironic
-    channel: latest/edge
+    channel: yoga/edge
   placement:
     charm: ch:placement
     num_units: 1
@@ -266,7 +266,7 @@ services:
     options:
       openstack-origin: *source
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   rabbitmq-server:
     charm: ch:rabbitmq-server
     num_units: 1
@@ -282,7 +282,7 @@ services:
     constraints: mem=2G
     options:
       openstack-origin: *source
-    channel: latest/edge
+    channel: yoga/edge
   ironic-conductor:
     charm: ch:ironic-conductor
     num_units: 1
@@ -293,15 +293,15 @@ services:
       disable-secure-erase: true
       use-ipxe: true
       enabled-network-interfaces: "flat, noop"
-    channel: latest/edge
+    channel: yoga/edge
   neutron-openvswitch:
     charm: ch:neutron-openvswitch
     num_units: 0
     options:
       bridge-mappings: physnet1:br-ex
-    channel: latest/edge
+    channel: yoga/edge
   neutron-ironic-agent:
-    charm: ../../../neutron-api-plugin-ironic
+    charm: ../../../neutron-api-plugin-ironic.charm
     num_units: 0
     options:
       openstack-origin: *source

src/tests/bundles/impish-xena.yaml

@@ -177,7 +177,7 @@ services:
       worker-multiplier: 0.25
     storage:
       block-devices: cinder,50G
-    channel: latest/edge
+    channel: yoga/edge
   ceph-radosgw:
     charm: ch:ceph-radosgw
     num_units: 1
@@ -185,7 +185,7 @@ services:
     options:
       source: *source
       namespace-tenants: True
-    channel: latest/edge
+    channel: quincy/edge
   ceph-mon:
     charm: ch:ceph-mon
     num_units: 3
@@ -193,7 +193,7 @@ services:
     options:
       expected-osd-count: 3
       source: *source
-    channel: latest/edge
+    channel: quincy/edge
   ceph-osd:
     charm: ch:ceph-osd
     num_units: 3
@@ -202,7 +202,7 @@ services:
       source: *source
     storage:
       osd-devices: 'cinder,30G'
-    channel: latest/edge
+    channel: quincy/edge
   glance:
     charm: ch:glance
     num_units: 1
@@ -210,7 +210,7 @@ services:
     options:
       openstack-origin: *source
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   keystone:
     charm: ch:keystone
     num_units: 1
@@ -218,7 +218,7 @@ services:
     options:
       openstack-origin: *source
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   neutron-api:
     charm: ch:neutron-api
     num_units: 1
@@ -229,7 +229,7 @@ services:
       openstack-origin: *source
       manage-neutron-plugin-legacy-mode: false
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   neutron-gateway:
     charm: ch:neutron-gateway
     num_units: 1
@@ -239,7 +239,7 @@ services:
       enable-isolated-metadata: true
       enable-metadata-network: true
       bridge-mappings: physnet1:br-ex
-    channel: latest/edge
+    channel: yoga/edge
   nova-cloud-controller:
     charm: ch:nova-cloud-controller
     num_units: 1
@@ -248,7 +248,7 @@ services:
       network-manager: Neutron
       openstack-origin: *source
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   nova-ironic:
     charm: ch:nova-compute
     num_units: 1
@@ -258,7 +258,7 @@ services:
       enable-resize: false
       openstack-origin: *source
       virt-type: ironic
-    channel: latest/edge
+    channel: yoga/edge
   placement:
     charm: ch:placement
     num_units: 1
@@ -266,7 +266,7 @@ services:
     options:
       openstack-origin: *source
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   rabbitmq-server:
     charm: ch:rabbitmq-server
     num_units: 1
@@ -282,7 +282,7 @@ services:
     constraints: mem=2G
     options:
       openstack-origin: *source
-    channel: latest/edge
+    channel: yoga/edge
   ironic-conductor:
     charm: ch:ironic-conductor
     num_units: 1
@@ -293,15 +293,15 @@ services:
       disable-secure-erase: true
       use-ipxe: true
       enabled-network-interfaces: "flat, noop"
-    channel: latest/edge
+    channel: yoga/edge
   neutron-openvswitch:
     charm: ch:neutron-openvswitch
     num_units: 0
     options:
       bridge-mappings: physnet1:br-ex
-    channel: latest/edge
+    channel: yoga/edge
   neutron-ironic-agent:
-    charm: ../../../neutron-api-plugin-ironic
+    charm: ../../../neutron-api-plugin-ironic.charm
     num_units: 0
     options:
       openstack-origin: *source

src/tests/bundles/jammy-yoga.yaml

@@ -177,7 +177,7 @@ services:
       worker-multiplier: 0.25
     storage:
       block-devices: cinder,50G
-    channel: latest/edge
+    channel: yoga/edge
   ceph-radosgw:
     charm: ch:ceph-radosgw
     num_units: 1
@@ -185,7 +185,7 @@ services:
     options:
       source: *source
       namespace-tenants: True
-    channel: latest/edge
+    channel: quincy/edge
   ceph-mon:
     charm: ch:ceph-mon
     num_units: 3
@@ -193,7 +193,7 @@ services:
     options:
       expected-osd-count: 3
       source: *source
-    channel: latest/edge
+    channel: quincy/edge
   ceph-osd:
     charm: ch:ceph-osd
     num_units: 3
@@ -202,7 +202,7 @@ services:
       source: *source
     storage:
       osd-devices: 'cinder,30G'
-    channel: latest/edge
+    channel: quincy/edge
   glance:
     charm: ch:glance
     num_units: 1
@@ -210,7 +210,7 @@ services:
     options:
       openstack-origin: *source
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   keystone:
     charm: ch:keystone
     num_units: 1
@@ -218,7 +218,7 @@ services:
     options:
       openstack-origin: *source
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   neutron-api:
     charm: ch:neutron-api
     num_units: 1
@@ -229,7 +229,7 @@ services:
       openstack-origin: *source
       manage-neutron-plugin-legacy-mode: false
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   neutron-gateway:
     charm: ch:neutron-gateway
     num_units: 1
@@ -239,7 +239,7 @@ services:
       enable-isolated-metadata: true
       enable-metadata-network: true
       bridge-mappings: physnet1:br-ex
-    channel: latest/edge
+    channel: yoga/edge
   nova-cloud-controller:
     charm: ch:nova-cloud-controller
     num_units: 1
@@ -248,7 +248,7 @@ services:
       network-manager: Neutron
       openstack-origin: *source
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   nova-ironic:
     charm: ch:nova-compute
     num_units: 1
@@ -258,7 +258,7 @@ services:
       enable-resize: false
       openstack-origin: *source
       virt-type: ironic
-    channel: latest/edge
+    channel: yoga/edge
   placement:
     charm: ch:placement
     num_units: 1
@@ -266,7 +266,7 @@ services:
     options:
       openstack-origin: *source
       worker-multiplier: 0.25
-    channel: latest/edge
+    channel: yoga/edge
   rabbitmq-server:
     charm: ch:rabbitmq-server
     num_units: 1
@@ -282,7 +282,7 @@ services:
     constraints: mem=2G
     options:
       openstack-origin: *source
-    channel: latest/edge
+    channel: yoga/edge
   ironic-conductor:
     charm: ch:ironic-conductor
     num_units: 1
@@ -293,15 +293,15 @@ services:
       disable-secure-erase: true
       use-ipxe: true
       enabled-network-interfaces: "flat, noop"
-    channel: latest/edge
+    channel: yoga/edge
   neutron-openvswitch:
     charm: ch:neutron-openvswitch
     num_units: 0
     options:
       bridge-mappings: physnet1:br-ex
-    channel: latest/edge
+    channel: yoga/edge
   neutron-ironic-agent:
-    charm: ../../../neutron-api-plugin-ironic
+    charm: ../../../neutron-api-plugin-ironic.charm
     num_units: 0
     options:
       openstack-origin: *source

src/tests/tests.yaml

@@ -17,10 +17,10 @@ smoke_bundles:
 target_deploy_status:
   vault:
     workload-status: blocked
-    workload-status-message: Vault needs to be initialized
+    workload-status-message-prefix: Vault needs to be initialized
   ironic-conductor:
     workload-status: blocked
-    workload-status-message: invalid enabled-deploy-interfaces config
+    workload-status-message-prefix: invalid enabled-deploy-interfaces config
   neutron-openvswitch:
     # NOTE(lourot): we deploy a bundle in which this subordinate charm has no
     # principle.

test-requirements.txt

@@ -51,3 +51,4 @@ pbr==5.6.0       # vault
 cryptography<3.4 # vault, keystone-saml-mellon
 lxml             # keystone-saml-mellon
 hvac             # vault, barbican-vault
+psutil           # cinder-lvm

tox.ini

@@ -37,11 +37,24 @@ setenv = VIRTUAL_ENV={envdir}
 passenv = http_proxy https_proxy INTERFACE_PATH LAYER_PATH JUJU_REPOSITORY
 install_command =
   {toxinidir}/pip.sh install {opts} {packages}
+allowlist_externals =
+    charmcraft
+    bash
+    tox
+    rename.sh
 deps =
     -r{toxinidir}/requirements.txt
 
 [testenv:build]
 basepython = python3
+deps = -r{toxinidir}/build-requirements.txt
+commands =
+    charmcraft clean
+    charmcraft -v build
+    {toxinidir}/rename.sh
+
+[testenv:build-reactive]
+basepython = python3
 commands =
     charm-build --log-level DEBUG --use-lock-file-branches -o {toxinidir}/build/builds src {posargs}
 
@@ -86,6 +99,18 @@ deps = flake8==3.9.2
        charm-tools==2.8.3
 commands = flake8 {posargs} src unit_tests
 
+[testenv:func-target]
+# Hack to get functional tests working in the charmcraft
+# world. We should fix this.
+basepython = python3
+passenv = HOME TERM CS_* OS_* TEST_*
+deps = -r{toxinidir}/src/test-requirements.txt
+changedir = {toxinidir}/src
+commands =
+  bash -c "if [ ! -f ../*.charm ]; then echo 'Charm does not exist. Run tox -e build'; exit 1; fi"
+  tox --version
+  tox -e func-target {posargs}
+
 [testenv:cover]
 # Technique based heavily upon
 # https://github.com/openstack/nova/blob/master/tox.ini