openstack
/
charm-glance-simplestreams-sync
Code Issues Proposed changes

Merge "Dynamically generate proxy settings for image syncs"

This commit is contained in:
Zuul 2021-07-23 14:38:45 +00:00 committed by Gerrit Code Review
parent 02ff4a0438 009c8a7b92
commit cf1f8bf725
5 changed files with 370 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
config.yaml
View File

@ -23,6 +23,13 @@ options:
on a local Apache server running on the unit and endpoints will be
registered referencing the local unit. This does not support HA
or TLS and is for testing purposes only.
ignore_proxy_for_object_store:
type: boolean
default: true
description: |
Controls whether Juju model proxy settings are going to be used
by sstream-mirror-glance when connecting to object-store endpoints
from the Keystone catalog.
frequency:
type: string
default: "daily"

159
files/glance_simplestreams_sync.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#
# Copyright 2014 Canonical Ltd.
#
@ -27,22 +27,28 @@ import atexit
import base64
import copy
import fcntl
import itertools
import logging
import os
import re
import shutil
import six
import sys
import subprocess
import sys
import tempfile
import time
import yaml
from keystoneclient import exceptions as keystone_exceptions
from keystoneclient.v2_0 import client as keystone_client
from keystoneclient.v3 import client as keystone_v3_client
from keystoneclient import exceptions as keystone_exceptions
if six.PY3:
from urllib import parse as urlparse
else:
import urlparse
def setup_logging():
def setup_file_logging():
logfilename = '/var/log/glance-simplestreams-sync.log'
if not os.path.exists(logfilename):
@ -60,10 +66,8 @@ def setup_logging():
logger.setLevel('DEBUG')
logger.addHandler(h)
return logger
log = setup_logging()
log = logging.getLogger()
KEYRING = '/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg'
CONF_FILE_DIR = '/etc/glance-simplestreams-sync'
@ -95,6 +99,12 @@ SSTREAM_LOG_FILE = os.path.join(SSTREAM_SNAP_COMMON,
CACERT_FILE = os.path.join(SSTREAM_SNAP_COMMON, 'cacert.pem')
SYSTEM_CACERT_FILE = '/etc/ssl/certs/ca-certificates.crt'
ENDPOINT_TYPES = [
'publicURL',
'adminURL',
'internalURL',
]
# TODOs:
# - allow people to specify their own policy, since they can specify
# their own mirrors.
@ -226,7 +236,7 @@ def set_openstack_env(id_conf, charm_conf):
os.environ['OS_TENANT_NAME'] = id_conf['admin_tenant_name']
def do_sync(charm_conf):
def do_sync(ksc, charm_conf):
# NOTE(beisner): the user_agent variable was an unused assignment (lint).
# It may be worth re-visiting its usage, intent and benefit with the
@ -234,6 +244,8 @@ def do_sync(charm_conf):
# and not assigning it since it is not currently utilized.
# user_agent = charm_conf.get("user_agent")
region_name = charm_conf['region']
for mirror_info in charm_conf['mirror_list']:
# NOTE: output directory must be under HOME
# or snap cannot access it for stream files
@ -241,7 +253,7 @@ def do_sync(charm_conf):
try:
log.info("Configuring sync for url {}".format(mirror_info))
content_id = charm_conf['content_id_template'].format(
region=charm_conf['region'])
region=region_name)
sync_command = [
"/snap/bin/simplestreams.sstream-mirror-glance",
@ -284,43 +296,135 @@ def do_sync(charm_conf):
]
sync_command += mirror_info['item_filters']
# Pass the current process' environment down along with proxy
# settings crafted for sstream-mirror-glance.
sstream_mirror_env = os.environ.copy()
sstream_mirror_env.update(get_sstream_mirror_proxy_env(
ksc, region_name,
charm_conf['ignore_proxy_for_object_store'],
))
log.info("calling sstream-mirror-glance")
log.debug("command: {}".format(" ".join(sync_command)))
subprocess.check_call(sync_command)
log.debug("command: %s", " ".join(sync_command))
log.debug("sstream-mirror environment: %s", sstream_mirror_env)
subprocess.check_call(sync_command, env=sstream_mirror_env)
if not charm_conf['use_swift']:
# Sync output directory to APACHE_DATA_DIR
subprocess.check_call([
'rsync', '-avz',
os.path.join(tmpdir, charm_conf['region'], 'streams'),
os.path.join(tmpdir, region_name, 'streams'),
APACHE_DATA_DIR
])
finally:
shutil.rmtree(tmpdir)
def update_product_streams_service(ksc, services, region):
"""
Updates URLs of product-streams endpoint to point to swift URLs.
"""
def get_sstream_mirror_proxy_env(ksc, region_name,
ignore_proxy_for_object_store=True):
'''Get proxy settings to be passed to sstreams-mirror-glance.
sstream-mirror-glance has multiple endpoints it needs to connect to:
1. Upstream image mirror (typically, an endpoint in public Internet);
2. Keystone (typically, a directly reachable endpoint);
3. Object storage (Swift) (typically a directly reachable endpoint).
4. Glance (typically, a directly reachable endpoint).
In a restricted environment where proxy settings have to be used for public
Internet connectivity we need to be explicit about hosts for which proxy
settings need to be used by sstream-mirror-glance. This function
dynamically builds a list of endpoints that need to be added to NO_PROXY
and optionally allows not including object storage endpoints into the
NO_PROXY list.
:param ksc: An instance of a Keystone client.
:type ksc: :class: `keystoneclient.v3.client.Client`
:param str region_name: A name of the region to retrieve endpoints for.
:param bool ignore_proxy_for_object_store: Do not include object-store
endpoints into NO_PROXY.
'''
proxy_settings = juju_proxy_settings()
if proxy_settings is None:
proxy_settings = {}
no_proxy_set = set()
else:
no_proxy_set = set(proxy_settings.get('NO_PROXY').split(','))
additional_hosts = set([
urlparse.urlparse(u).hostname for u in itertools.chain(
get_service_endpoints(ksc, 'identity', region_name).values(),
get_service_endpoints(ksc, 'image', region_name).values(),
get_service_endpoints(ksc, 'object-store', region_name).values()
if ignore_proxy_for_object_store else [],
)])
no_proxy = ','.join(no_proxy_set | additional_hosts)
proxy_settings['NO_PROXY'] = no_proxy
proxy_settings['no_proxy'] = no_proxy
return proxy_settings
def update_product_streams_service(ksc, services, region):
"""Updates URLs of product-streams endpoint to point to swift URLs."""
object_store_endpoints = get_service_endpoints(ksc, 'object-store', region)
for endpoint_type in ENDPOINT_TYPES:
object_store_endpoints[endpoint_type] += "/{}".format(SWIFT_DATA_DIR)
publicURL, internalURL, adminURL = (object_store_endpoints[t]
for t in ENDPOINT_TYPES)
# Update the relation to keystone to update the catalog URLs
update_endpoint_urls(
region,
publicURL,
internalURL,
adminURL,
)
def get_service_endpoints(ksc, service_type, region_name):
"""Get endpoints for a given service type from the Keystone catalog.
:param ksc: An instance of a Keystone client.
:type ksc: :class: `keystoneclient.v3.client.Client`
:param str service_type: An endpoint service type to use.
:param str region_name: A name of the region to retrieve endpoints for.
:raises :class: `keystone_exceptions.EndpointNotFound`
"""
try:
catalog = {
endpoint_type: ksc.service_catalog.url_for(
service_type='object-store', endpoint_type=endpoint_type)
service_type=service_type, endpoint_type=endpoint_type,
region_name=region_name)
for endpoint_type in ['publicURL', 'internalURL', 'adminURL']}
except keystone_exceptions.EndpointNotFound as e:
log.warning("could not retrieve swift endpoint, not updating "
"product-streams endpoint: {}".format(e))
except keystone_exceptions.EndpointNotFound:
log.error('could not retrieve any {} endpoints'.format(service_type))
raise
return catalog
for endpoint_type in ['publicURL', 'internalURL']:
catalog[endpoint_type] += "/{}".format(SWIFT_DATA_DIR)
# Update the relation to keystone to update the catalog URLs
update_endpoint_urls(region, catalog['publicURL'],
catalog['adminURL'],
catalog['internalURL'])
def juju_proxy_settings():
"""Get proxy settings from Juju environment.
Get charm proxy settings from environment variables that correspond to
juju-http-proxy, juju-https-proxy juju-no-proxy (available as of 2.4.2, see
lp:1782236) or the legacy unprefixed settings.
:rtype: None | dict[str, str]
"""
# Get proxy settings from the environment variables set by Juju.
juju_settings = {
m.groupdict()['var']: m.groupdict()['val']
for m in re.finditer(
'^((JUJU_CHARM_)?(?P<var>(HTTP|HTTPS|NO)_PROXY))=(?P<val>.*)$',
juju_run_cmd(['env']), re.MULTILINE)
}
proxy_settings = {}
for var in ['HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY']:
var_val = juju_settings.get(var)
if var_val:
proxy_settings[var] = var_val
proxy_settings[var.lower()] = var_val
return proxy_settings if proxy_settings else None
def juju_run_cmd(cmd):
@ -430,7 +534,7 @@ def main():
log.info("Beginning image sync")
status_set('maintenance', 'Synchronising images')
do_sync(charm_conf)
do_sync(ksc, charm_conf)
ts = time.strftime("%x %X")
# "Unit is ready" is one of approved message prefixes
# Prefix the message with it will help zaza to understand the status.
@ -459,4 +563,5 @@ def main():
if __name__ == "__main__":
setup_file_logging()
sys.exit(main())

2
hooks/hooks.py
View File

@ -174,6 +174,8 @@ class MirrorsConfigServiceContext(OSContextGenerator):
name_prefix=config['name_prefix'],
content_id_template=config['content_id_template'],
use_swift=config['use_swift'],
ignore_proxy_for_object_store=config[
'ignore_proxy_for_object_store'],
region=config['region'],
cloud_name=config['cloud_name'],
user_agent=config['user_agent'],

1
templates/mirrors.yaml
View File

@ -3,6 +3,7 @@ user_agent: {{ user_agent }}
modify_hook_scripts: {{ modify_hook_scripts }}
name_prefix: {{ name_prefix }}
use_swift: {{ use_swift }}
ignore_proxy_for_object_store: {{ ignore_proxy_for_object_store }}
region: {{ region }}
cloud_name: {{ cloud_name }}
content_id_template: {{ content_id_template }}

228
unit_tests/test_glance_simplestreams_sync.py Normal file
View File

@ -0,0 +1,228 @@
#!/usr/bin/env python3
'''
Copyright 2021 Canonical Ltd.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
import files.glance_simplestreams_sync as gss
import mock
import unittest
from keystoneclient import exceptions as keystone_exceptions
class TestGlanceSimpleStreamsSync(unittest.TestCase):
def setUp(self):
self.maxDiff = 4096
@mock.patch('files.glance_simplestreams_sync.juju_run_cmd')
def test_proxy_settings(self, juju_run_cmd):
juju_run_cmd.return_value = '''
LANG=C.UTF-8
JUJU_CONTEXT_ID=glance-simplestreams-sync/0-run-commands-3325280900519425661
JUJU_CHARM_HTTP_PROXY=http://squid.internal:3128
JUJU_CHARM_HTTPS_PROXY=https://squid.internal:3128
JUJU_CHARM_NO_PROXY=127.0.0.1,localhost,::1
'''
self.assertEqual(gss.juju_proxy_settings(), {
"HTTP_PROXY": "http://squid.internal:3128",
"HTTPS_PROXY": "https://squid.internal:3128",
"NO_PROXY": "127.0.0.1,localhost,::1",
"http_proxy": "http://squid.internal:3128",
"https_proxy": "https://squid.internal:3128",
"no_proxy": "127.0.0.1,localhost,::1",
})
@mock.patch('files.glance_simplestreams_sync.juju_run_cmd')
def test_legacy_proxy_settings(self, juju_run_cmd):
juju_run_cmd.return_value = '''
LANG=C.UTF-8
JUJU_CONTEXT_ID=glance-simplestreams-sync/0-run-commands-3325280900519425661
HTTP_PROXY=http://squid.internal:3128
HTTPS_PROXY=https://squid.internal:3128
NO_PROXY=127.0.0.1,localhost,::1
'''
self.assertEqual(gss.juju_proxy_settings(), {
"HTTP_PROXY": "http://squid.internal:3128",
"HTTPS_PROXY": "https://squid.internal:3128",
"NO_PROXY": "127.0.0.1,localhost,::1",
"http_proxy": "http://squid.internal:3128",
"https_proxy": "https://squid.internal:3128",
"no_proxy": "127.0.0.1,localhost,::1",
})
@mock.patch('files.glance_simplestreams_sync.juju_run_cmd')
def test_proxy_settings_not_set(self, juju_run_cmd):
juju_run_cmd.return_value = '''
LANG=C.UTF-8
JUJU_CONTEXT_ID=glance-simplestreams-sync/0-run-commands-3325280900519425661
'''
self.assertEqual(gss.juju_proxy_settings(), None)
@mock.patch('files.glance_simplestreams_sync.get_service_endpoints')
@mock.patch('files.glance_simplestreams_sync.juju_proxy_settings')
def test_get_sstream_mirror_proxy_env(self,
juju_proxy_settings,
get_service_endpoints):
# Use a side effect instead of return value to avoid modification of
# the same dict in different invocations of the tested function.
def juju_proxy_settings_side_effect():
return {
"HTTP_PROXY": "http://squid.internal:3128",
"HTTPS_PROXY": "https://squid.internal:3128",
"NO_PROXY": "127.0.0.1,localhost,::1",
"http_proxy": "http://squid.internal:3128",
"https_proxy": "https://squid.internal:3128",
"no_proxy": "127.0.0.1,localhost,::1",
}
juju_proxy_settings.side_effect = juju_proxy_settings_side_effect
def get_service_endpoints_side_effect(ksc, service_type, region_name):
return {
'identity': {
'publicURL': 'https://192.0.2.42:5000/v3',
'internalURL': 'https://192.0.2.43:5000/v3',
'adminURL': 'https://192.0.2.44:35357/v3',
},
'image': {
'publicURL': 'https://192.0.2.45:9292',
'internalURL': 'https://192.0.2.45:9292',
'adminURL': 'https://192.0.2.47:9292',
},
'object-store': {
'publicURL': 'https://192.0.2.90:443/swift/v1',
'internalURL': 'https://192.0.2.90:443/swift/v1',
'adminURL': 'https://192.0.2.90:443/swift',
},
}[service_type]
get_service_endpoints.side_effect = get_service_endpoints_side_effect
# Besides checking for proxy settings being set, make sure that
# object-store endpoints are added to NO_PROXY by default or when
# explicitly asked for.
for proxy_env in [
gss.get_sstream_mirror_proxy_env(
mock.MagicMock(), 'TestRegion'),
gss.get_sstream_mirror_proxy_env(
mock.MagicMock(), 'TestRegion',
ignore_proxy_for_object_store=True)]:
self.assertEqual(proxy_env['HTTP_PROXY'],
'http://squid.internal:3128')
self.assertEqual(proxy_env['http_proxy'],
'http://squid.internal:3128')
self.assertEqual(proxy_env['HTTPS_PROXY'],
'https://squid.internal:3128')
self.assertEqual(proxy_env['https_proxy'],
'https://squid.internal:3128')
no_proxy_set = set(['127.0.0.1', 'localhost', '::1', '192.0.2.42',
'192.0.2.43', '192.0.2.44', '192.0.2.45',
'192.0.2.47', '192.0.2.90'])
self.assertEqual(set(proxy_env['NO_PROXY'].split(',')),
no_proxy_set)
self.assertEqual(set(proxy_env['no_proxy'].split(',')),
no_proxy_set)
# Make sure that object-store endpoints are not included into
# NO_PROXY when this is explicitly being asked for. In this case
# the set of expected addresses in NO_PROXY should exclude 192.0.2.90.
proxy_env = gss.get_sstream_mirror_proxy_env(
mock.MagicMock(),
'TestRegion', ignore_proxy_for_object_store=False)
self.assertEqual(proxy_env['HTTP_PROXY'], 'http://squid.internal:3128')
self.assertEqual(proxy_env['http_proxy'], 'http://squid.internal:3128')
self.assertEqual(proxy_env['HTTPS_PROXY'],
'https://squid.internal:3128')
self.assertEqual(proxy_env['https_proxy'],
'https://squid.internal:3128')
no_proxy_set_no_obj = set(['127.0.0.1', 'localhost', '::1',
'192.0.2.42', '192.0.2.43', '192.0.2.44',
'192.0.2.45', '192.0.2.47'])
self.assertEqual(set(proxy_env['NO_PROXY'].split(',')),
no_proxy_set_no_obj)
self.assertEqual(set(proxy_env['no_proxy'].split(',')),
no_proxy_set_no_obj)
def no_juju_proxy_settings_side_effect():
return None
juju_proxy_settings.side_effect = no_juju_proxy_settings_side_effect
# Make sure that even if Juju does not have any proxy settings set,
# via the model, we are still adding endpoints to NO_PROXY for
# sstream-mirror-glance invocations because settings might be sourced
# from other files (see glance-simplestreams-sync.sh).
proxy_env = gss.get_sstream_mirror_proxy_env(
mock.MagicMock(),
'TestRegion', ignore_proxy_for_object_store=False)
no_proxy_set_no_obj = set(['192.0.2.42', '192.0.2.43', '192.0.2.44',
'192.0.2.45', '192.0.2.47'])
self.assertEqual(set(proxy_env['NO_PROXY'].split(',')),
no_proxy_set_no_obj)
self.assertEqual(set(proxy_env['no_proxy'].split(',')),
no_proxy_set_no_obj)
def test_get_service_endpoints(self):
def url_for_side_effect(service_type, endpoint_type, region_name):
return {
'TestRegion': {
'identity': {
'publicURL': 'https://10.5.2.42:443/swift/v1',
'internalURL': 'https://10.5.2.42:443/swift/v1',
'adminURL': 'https://10.5.2.42:443/swift/v1',
},
'image': {
'publicURL': 'https://10.5.2.43:443/swift/v1',
'internalURL': 'https://10.5.2.43:443/swift/v1',
'adminURL': 'https://10.5.2.43:443/swift/v1',
},
'object-store': {
'publicURL': 'https://10.5.2.44:443/swift/v1',
'internalURL': 'https://10.5.2.44:443/swift/v1',
'adminURL': 'https://10.5.2.44:443/swift/v1',
},
}
}[region_name][service_type][endpoint_type]
ksc = mock.MagicMock()
ksc.service_catalog.url_for.side_effect = url_for_side_effect
self.assertEqual(
gss.get_service_endpoints(ksc, 'identity', 'TestRegion'), {
'publicURL': 'https://10.5.2.42:443/swift/v1',
'internalURL': 'https://10.5.2.42:443/swift/v1',
'adminURL': 'https://10.5.2.42:443/swift/v1',
}
)
self.assertEqual(
gss.get_service_endpoints(ksc, 'image', 'TestRegion'), {
'publicURL': 'https://10.5.2.43:443/swift/v1',
'internalURL': 'https://10.5.2.43:443/swift/v1',
'adminURL': 'https://10.5.2.43:443/swift/v1',
}
)
self.assertEqual(
gss.get_service_endpoints(ksc, 'object-store', 'TestRegion'), {
'publicURL': 'https://10.5.2.44:443/swift/v1',
'internalURL': 'https://10.5.2.44:443/swift/v1',
'adminURL': 'https://10.5.2.44:443/swift/v1',
}
)
ksc.service_catalog.url_for.side_effect = mock.MagicMock(
side_effect=keystone_exceptions.EndpointException('foo'))
with self.assertRaises(keystone_exceptions.EndpointException):
gss.get_service_endpoints(ksc, 'test', 'TestRegion')