Allow local hosting of product-streams

When use_swift=False in addition to syncing images into glance, also
host the product streams metadata using apache.

Change-Id: I6aad2fee3670f575014d0845f3bd7aae6fa06785
Closes-Bug: #1840830

hooks/hooks.py

@@ -65,6 +65,11 @@ CRON_POLL_FILEPATH = os.path.join(CRON_D, CRON_POLL_FILENAME)
 
 ERR_FILE_EXISTS = 17
 
+PACKAGES = ['python-simplestreams', 'python-glanceclient',
+            'python-yaml', 'python-keystoneclient',
+            'python-kombu',
+            'python-swiftclient', 'ubuntu-cloudimage-keyring']
+
 hooks = hookenv.Hooks()
 
 
@@ -248,12 +253,14 @@ def install():
                 return
             os.mkdir(directory)
 
+    _packages = PACKAGES
+    if not hookenv.config("use_swift"):
+        hookenv.log('Configuring for local hosting of product stream.')
+        _packages += ["apache2"]
+
     apt_update(fatal=True)
 
-    apt_install(packages=['python-simplestreams', 'python-glanceclient',
-                          'python-yaml', 'python-keystoneclient',
-                          'python-kombu',
-                          'python-swiftclient', 'ubuntu-cloudimage-keyring'])
+    apt_install(_packages)
 
     hookenv.log('end install hook.')
 

scripts/glance-simplestreams-sync.py

@@ -61,6 +61,7 @@ import keystoneclient.exceptions as keystone_exceptions
 import kombu
 from simplestreams.mirrors import glance, UrlMirrorReader
 from simplestreams.objectstores.swift import SwiftObjectStore
+from simplestreams.objectstores import FileStore
 from simplestreams.util import read_signed, path_from_mirror_url
 import sys
 import time
@@ -82,6 +83,9 @@ SYNC_RUNNING_FLAG_FILE_NAME = os.path.join(PID_FILE_DIR,
 # it is.
 SWIFT_DATA_DIR = 'simplestreams/data/'
 
+# When running local apache for product-streams use path to place indexes.
+APACHE_DATA_DIR = '/var/www/html'
+
 PRODUCT_STREAMS_SERVICE_NAME = 'image-stream'
 PRODUCT_STREAMS_SERVICE_TYPE = 'product-streams'
 PRODUCT_STREAMS_SERVICE_DESC = 'Ubuntu Product Streams'
@@ -254,7 +258,8 @@ def do_sync(charm_conf, status_exchange):
         if charm_conf['use_swift']:
             store = SwiftObjectStore(SWIFT_DATA_DIR)
         else:
-            store = None
+            # Use the local apache server to serve product streams
+            store = FileStore(prefix=APACHE_DATA_DIR)
 
         content_id = charm_conf['content_id_template'].format(
             region=charm_conf['region'])

tests/basic_deployment.py

@@ -19,7 +19,9 @@ Basic glance-simplestreams-sync functional tests.
 """
 
 import amulet
+import json
 import re
+import requests
 import time
 
 from charmhelpers.contrib.openstack.amulet.deployment import (
@@ -159,7 +161,7 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment):
         # Authenticate admin with glance endpoint
         self.glance = u.authenticate_glance_admin(self.keystone)
 
-    def test_001_wait_for_image_sync(self):
+    def test_010_wait_for_image_sync(self):
         """Wait for images to be synced. Expect at least one."""
 
         max_image_wait = 600
@@ -237,3 +239,25 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment):
             openstack_release=self._get_openstack_release())
         if ret:
             amulet.raise_status(amulet.FAIL, msg=ret)
+
+    def test_110_local_product_stream(self):
+        """Verify that the local product stream is accessible and has data"""
+        u.log.debug('Checking local product streams...')
+        _expected = ['com.ubuntu.cloud:server:14.04:amd64',
+                     'com.ubuntu.cloud:server:16.04:amd64',
+                     'com.ubuntu.cloud:server:18.04:amd64']
+        _uri = "streams/v1/auto.sync.json"
+        _key = "url"
+        if self._get_openstack_release() <= self.xenial_pike:
+            _key = "publicURL"
+
+        _catalog = self.keystone.service_catalog.get_endpoints()
+        _ps_interface = _catalog["product-streams"][0][_key]
+        _url = "{}/{}".format(_ps_interface, _uri)
+        _client = requests.session()
+        _json_data = _client.get(_url).text
+        _product_streams = json.loads(_json_data)
+
+        for image in _expected:
+            assert image in _product_streams["products"].keys()
+        u.log.debug("Local product stream successful")