openstack/charm-glance-simplestreams-sync
Code Issues Proposed changes

Dynamically generate proxy settings for image syncs

Browse Source 
sstream-mirror-glance has several endpoints it needs to talk to:

* Image mirrors - typically, public Internet endpoints;
* Keystone - typically, a directly reachable endpoint;
* Glance - typically, a directly reachable endpoint;
* Object store (Swift) - typically, a directly reachable endpoint but
  sometimes it may be deployed externally and added to the region
  catalog in Keystone (in which case it might be accessible via a proxy
  only).

While sstream-mirror-glance does not support specifying proxy settings
for individual directions, since we know all of them based on the
Keystone catalog, a list of endpoints to add to NO_PROXY environment
variable can be generated dynamically.

The complication is that image syncs are periodically done via a cron
job so a juju-run invocation is needed to retrieve relevant proxy
settings from model-config at each invocation of the synchronization
script.

Additionally, the charm is long-lived so there may be some environments
that rely on legacy proxy settings. This change accounts for that and
acts both on juju-prefixed (new) and unprefixed (legacy) proxy settings.

Whether to use proxy settings for connections to the object store API
is controlled by a charm option which the script is made to react to.
Proxy settings are ignored for object store connections by default.

Closes-Bug: #1843486
Change-Id: Ib1fc5d2eebf43d5f98bb8ee405a3799802c8b8dc
This commit is contained in:
Dmitrii Shcherbakov 2021-07-15 20:49:16 +03:00
parent 787a9c5ae9
commit 009c8a7b92
8 changed files with 374 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
config.yaml
View File

@ -23,6 +23,13 @@ options:
on a local Apache server running on the unit and endpoints will be
registered referencing the local unit. This does not support HA
or TLS and is for testing purposes only.
ignore_proxy_for_object_store:
type: boolean
default: true
description: |
Controls whether Juju model proxy settings are going to be used
by sstream-mirror-glance when connecting to object-store endpoints
from the Keystone catalog.
frequency:
type: string
default: "daily"

159
files/glance_simplestreams_sync.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#
# Copyright 2014 Canonical Ltd.
#
@ -27,22 +27,28 @@ import atexit
import base64
import copy
import fcntl
import itertools
import logging
import os
import re
import shutil
import six
import sys
import subprocess
import sys
import tempfile
import time
import yaml
from keystoneclient import exceptions as keystone_exceptions
from keystoneclient.v2_0 import client as keystone_client
from keystoneclient.v3 import client as keystone_v3_client
from keystoneclient import exceptions as keystone_exceptions
if six.PY3:
from urllib import parse as urlparse
else:
import urlparse
def setup_logging():
def setup_file_logging():
logfilename = '/var/log/glance-simplestreams-sync.log'
if not os.path.exists(logfilename):
@ -60,10 +66,8 @@ def setup_logging():
logger.setLevel('DEBUG')
logger.addHandler(h)
return logger
log = setup_logging()
log = logging.getLogger()
KEYRING = '/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg'
CONF_FILE_DIR = '/etc/glance-simplestreams-sync'
@ -95,6 +99,12 @@ SSTREAM_LOG_FILE = os.path.join(SSTREAM_SNAP_COMMON,
CACERT_FILE = os.path.join(SSTREAM_SNAP_COMMON, 'cacert.pem')
SYSTEM_CACERT_FILE = '/etc/ssl/certs/ca-certificates.crt'
ENDPOINT_TYPES = [
'publicURL',
'adminURL',
'internalURL',
]
# TODOs:
# - allow people to specify their own policy, since they can specify
# their own mirrors.
@ -226,7 +236,7 @@ def set_openstack_env(id_conf, charm_conf):
os.environ['OS_TENANT_NAME'] = id_conf['admin_tenant_name']
def do_sync(charm_conf):
def do_sync(ksc, charm_conf):
# NOTE(beisner): the user_agent variable was an unused assignment (lint).
# It may be worth re-visiting its usage, intent and benefit with the
@ -234,6 +244,8 @@ def do_sync(charm_conf):
# and not assigning it since it is not currently utilized.
# user_agent = charm_conf.get("user_agent")
region_name = charm_conf['region']
for mirror_info in charm_conf['mirror_list']:
# NOTE: output directory must be under HOME
# or snap cannot access it for stream files
@ -241,7 +253,7 @@ def do_sync(charm_conf):
try:
log.info("Configuring sync for url {}".format(mirror_info))
content_id = charm_conf['content_id_template'].format(
region=charm_conf['region'])
region=region_name)
sync_command = [
"/snap/bin/simplestreams.sstream-mirror-glance",
@ -284,43 +296,135 @@ def do_sync(charm_conf):
]
sync_command += mirror_info['item_filters']
# Pass the current process' environment down along with proxy
# settings crafted for sstream-mirror-glance.
sstream_mirror_env = os.environ.copy()
sstream_mirror_env.update(get_sstream_mirror_proxy_env(
ksc, region_name,
charm_conf['ignore_proxy_for_object_store'],
))
log.info("calling sstream-mirror-glance")
log.debug("command: {}".format(" ".join(sync_command)))
subprocess.check_call(sync_command)
log.debug("command: %s", " ".join(sync_command))
log.debug("sstream-mirror environment: %s", sstream_mirror_env)
subprocess.check_call(sync_command, env=sstream_mirror_env)
if not charm_conf['use_swift']:
# Sync output directory to APACHE_DATA_DIR
subprocess.check_call([
'rsync', '-avz',
os.path.join(tmpdir, charm_conf['region'], 'streams'),
os.path.join(tmpdir, region_name, 'streams'),
APACHE_DATA_DIR
])
finally:
shutil.rmtree(tmpdir)
def update_product_streams_service(ksc, services, region):
"""
Updates URLs of product-streams endpoint to point to swift URLs.
"""
def get_sstream_mirror_proxy_env(ksc, region_name,
ignore_proxy_for_object_store=True):
'''Get proxy settings to be passed to sstreams-mirror-glance.
sstream-mirror-glance has multiple endpoints it needs to connect to:
1. Upstream image mirror (typically, an endpoint in public Internet);
2. Keystone (typically, a directly reachable endpoint);
3. Object storage (Swift) (typically a directly reachable endpoint).
4. Glance (typically, a directly reachable endpoint).
In a restricted environment where proxy settings have to be used for public
Internet connectivity we need to be explicit about hosts for which proxy
settings need to be used by sstream-mirror-glance. This function
dynamically builds a list of endpoints that need to be added to NO_PROXY
and optionally allows not including object storage endpoints into the
NO_PROXY list.
:param ksc: An instance of a Keystone client.
:type ksc: :class: `keystoneclient.v3.client.Client`
:param str region_name: A name of the region to retrieve endpoints for.
:param bool ignore_proxy_for_object_store: Do not include object-store
endpoints into NO_PROXY.
'''
proxy_settings = juju_proxy_settings()
if proxy_settings is None:
proxy_settings = {}
no_proxy_set = set()
else:
no_proxy_set = set(proxy_settings.get('NO_PROXY').split(','))
additional_hosts = set([
urlparse.urlparse(u).hostname for u in itertools.chain(
get_service_endpoints(ksc, 'identity', region_name).values(),
get_service_endpoints(ksc, 'image', region_name).values(),
get_service_endpoints(ksc, 'object-store', region_name).values()
if ignore_proxy_for_object_store else [],
)])
no_proxy = ','.join(no_proxy_set | additional_hosts)
proxy_settings['NO_PROXY'] = no_proxy
proxy_settings['no_proxy'] = no_proxy
return proxy_settings
def update_product_streams_service(ksc, services, region):
"""Updates URLs of product-streams endpoint to point to swift URLs."""
object_store_endpoints = get_service_endpoints(ksc, 'object-store', region)
for endpoint_type in ENDPOINT_TYPES:
object_store_endpoints[endpoint_type] += "/{}".format(SWIFT_DATA_DIR)
publicURL, internalURL, adminURL = (object_store_endpoints[t]
for t in ENDPOINT_TYPES)
# Update the relation to keystone to update the catalog URLs
update_endpoint_urls(
region,
publicURL,
internalURL,
adminURL,
)
def get_service_endpoints(ksc, service_type, region_name):
"""Get endpoints for a given service type from the Keystone catalog.
:param ksc: An instance of a Keystone client.
:type ksc: :class: `keystoneclient.v3.client.Client`
:param str service_type: An endpoint service type to use.
:param str region_name: A name of the region to retrieve endpoints for.
:raises :class: `keystone_exceptions.EndpointNotFound`
"""
try:
catalog = {
endpoint_type: ksc.service_catalog.url_for(
service_type='object-store', endpoint_type=endpoint_type)
service_type=service_type, endpoint_type=endpoint_type,
region_name=region_name)
for endpoint_type in ['publicURL', 'internalURL', 'adminURL']}
except keystone_exceptions.EndpointNotFound as e:
log.warning("could not retrieve swift endpoint, not updating "
"product-streams endpoint: {}".format(e))
except keystone_exceptions.EndpointNotFound:
log.error('could not retrieve any {} endpoints'.format(service_type))
raise
return catalog
for endpoint_type in ['publicURL', 'internalURL']:
catalog[endpoint_type] += "/{}".format(SWIFT_DATA_DIR)
# Update the relation to keystone to update the catalog URLs
update_endpoint_urls(region, catalog['publicURL'],
catalog['adminURL'],
catalog['internalURL'])
def juju_proxy_settings():
"""Get proxy settings from Juju environment.
Get charm proxy settings from environment variables that correspond to
juju-http-proxy, juju-https-proxy juju-no-proxy (available as of 2.4.2, see
lp:1782236) or the legacy unprefixed settings.
:rtype: None | dict[str, str]
"""
# Get proxy settings from the environment variables set by Juju.
juju_settings = {
m.groupdict()['var']: m.groupdict()['val']
for m in re.finditer(
'^((JUJU_CHARM_)?(?P<var>(HTTP|HTTPS|NO)_PROXY))=(?P<val>.*)$',
juju_run_cmd(['env']), re.MULTILINE)
}
proxy_settings = {}
for var in ['HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY']:
var_val = juju_settings.get(var)
if var_val:
proxy_settings[var] = var_val
proxy_settings[var.lower()] = var_val
return proxy_settings if proxy_settings else None
def juju_run_cmd(cmd):
@ -430,7 +534,7 @@ def main():
log.info("Beginning image sync")
status_set('maintenance', 'Synchronising images')
do_sync(charm_conf)
do_sync(ksc, charm_conf)
ts = time.strftime("%x %X")
# "Unit is ready" is one of approved message prefixes
# Prefix the message with it will help zaza to understand the status.
@ -459,4 +563,5 @@ def main():
if __name__ == "__main__":
setup_file_logging()
sys.exit(main())

2
hooks/hooks.py
View File

@ -174,6 +174,8 @@ class MirrorsConfigServiceContext(OSContextGenerator):
name_prefix=config['name_prefix'],
content_id_template=config['content_id_template'],
use_swift=config['use_swift'],
ignore_proxy_for_object_store=config[
'ignore_proxy_for_object_store'],
region=config['region'],
cloud_name=config['cloud_name'],
user_agent=config['user_agent'],

2
requirements.txt
View File

@ -7,7 +7,7 @@
# requirements. They are intertwined. Also, Zaza itself should specify
# all of its own requirements and if it doesn't, fix it there.
#
pbr>=1.8.0,<1.9.0
pbr==5.6.0
simplejson>=2.2.0
netifaces>=0.10.4

1
templates/mirrors.yaml
View File

@ -3,6 +3,7 @@ user_agent: {{ user_agent }}
modify_hook_scripts: {{ modify_hook_scripts }}
name_prefix: {{ name_prefix }}
use_swift: {{ use_swift }}
ignore_proxy_for_object_store: {{ ignore_proxy_for_object_store }}
region: {{ region }}
cloud_name: {{ cloud_name }}
content_id_template: {{ content_id_template }}

8
test-requirements.txt
View File

@ -8,11 +8,6 @@
# all of its own requirements and if it doesn't, fix it there.
#
setuptools<50.0.0 # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85
charm-tools>=2.4.4
# Workaround until https://github.com/juju/charm-tools/pull/589 gets
# published
keyring<21
requests>=2.18.4
@ -21,7 +16,6 @@ requests>=2.18.4
mock>=1.2,<4.0.0; python_version < '3.6'
mock>=1.2; python_version >= '3.6'
flake8>=2.2.4
stestr>=2.2.0
# Dependency of stestr. Workaround for
@ -42,7 +36,7 @@ oslo.utils<=3.41.0;python_version<'3.6'
coverage>=4.5.2
pyudev # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking)
git+https://github.com/openstack-charmers/zaza.git#egg=zaza;python_version>='3.0'
git+https://github.com/openstack-charmers/zaza.git#egg=zaza
git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack
# Needed for charm-glance:

4
tox.ini
View File

@ -65,8 +65,8 @@ deps = -r{toxinidir}/requirements.txt
[testenv:pep8]
basepython = python3
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
deps = flake8==3.9.2
charm-tools==2.8.3
commands = flake8 {posargs} hooks unit_tests tests actions lib files
charm-proof

228
unit_tests/test_glance_simplestreams_sync.py Normal file
View File

@ -0,0 +1,228 @@
#!/usr/bin/env python3
'''
Copyright 2021 Canonical Ltd.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
import files.glance_simplestreams_sync as gss
import mock
import unittest
from keystoneclient import exceptions as keystone_exceptions
class TestGlanceSimpleStreamsSync(unittest.TestCase):
def setUp(self):
self.maxDiff = 4096
@mock.patch('files.glance_simplestreams_sync.juju_run_cmd')
def test_proxy_settings(self, juju_run_cmd):
juju_run_cmd.return_value = '''
LANG=C.UTF-8
JUJU_CONTEXT_ID=glance-simplestreams-sync/0-run-commands-3325280900519425661
JUJU_CHARM_HTTP_PROXY=http://squid.internal:3128
JUJU_CHARM_HTTPS_PROXY=https://squid.internal:3128
JUJU_CHARM_NO_PROXY=127.0.0.1,localhost,::1
'''
self.assertEqual(gss.juju_proxy_settings(), {
"HTTP_PROXY": "http://squid.internal:3128",
"HTTPS_PROXY": "https://squid.internal:3128",
"NO_PROXY": "127.0.0.1,localhost,::1",
"http_proxy": "http://squid.internal:3128",
"https_proxy": "https://squid.internal:3128",
"no_proxy": "127.0.0.1,localhost,::1",
})
@mock.patch('files.glance_simplestreams_sync.juju_run_cmd')
def test_legacy_proxy_settings(self, juju_run_cmd):
juju_run_cmd.return_value = '''
LANG=C.UTF-8
JUJU_CONTEXT_ID=glance-simplestreams-sync/0-run-commands-3325280900519425661
HTTP_PROXY=http://squid.internal:3128
HTTPS_PROXY=https://squid.internal:3128
NO_PROXY=127.0.0.1,localhost,::1
'''
self.assertEqual(gss.juju_proxy_settings(), {
"HTTP_PROXY": "http://squid.internal:3128",
"HTTPS_PROXY": "https://squid.internal:3128",
"NO_PROXY": "127.0.0.1,localhost,::1",
"http_proxy": "http://squid.internal:3128",
"https_proxy": "https://squid.internal:3128",
"no_proxy": "127.0.0.1,localhost,::1",
})
@mock.patch('files.glance_simplestreams_sync.juju_run_cmd')
def test_proxy_settings_not_set(self, juju_run_cmd):
juju_run_cmd.return_value = '''
LANG=C.UTF-8
JUJU_CONTEXT_ID=glance-simplestreams-sync/0-run-commands-3325280900519425661
'''
self.assertEqual(gss.juju_proxy_settings(), None)
@mock.patch('files.glance_simplestreams_sync.get_service_endpoints')
@mock.patch('files.glance_simplestreams_sync.juju_proxy_settings')
def test_get_sstream_mirror_proxy_env(self,
juju_proxy_settings,
get_service_endpoints):
# Use a side effect instead of return value to avoid modification of
# the same dict in different invocations of the tested function.
def juju_proxy_settings_side_effect():
return {
"HTTP_PROXY": "http://squid.internal:3128",
"HTTPS_PROXY": "https://squid.internal:3128",
"NO_PROXY": "127.0.0.1,localhost,::1",
"http_proxy": "http://squid.internal:3128",
"https_proxy": "https://squid.internal:3128",
"no_proxy": "127.0.0.1,localhost,::1",
}
juju_proxy_settings.side_effect = juju_proxy_settings_side_effect
def get_service_endpoints_side_effect(ksc, service_type, region_name):
return {
'identity': {
'publicURL': 'https://192.0.2.42:5000/v3',
'internalURL': 'https://192.0.2.43:5000/v3',
'adminURL': 'https://192.0.2.44:35357/v3',
},
'image': {
'publicURL': 'https://192.0.2.45:9292',
'internalURL': 'https://192.0.2.45:9292',
'adminURL': 'https://192.0.2.47:9292',
},
'object-store': {
'publicURL': 'https://192.0.2.90:443/swift/v1',
'internalURL': 'https://192.0.2.90:443/swift/v1',
'adminURL': 'https://192.0.2.90:443/swift',
},
}[service_type]
get_service_endpoints.side_effect = get_service_endpoints_side_effect
# Besides checking for proxy settings being set, make sure that
# object-store endpoints are added to NO_PROXY by default or when
# explicitly asked for.
for proxy_env in [
gss.get_sstream_mirror_proxy_env(
mock.MagicMock(), 'TestRegion'),
gss.get_sstream_mirror_proxy_env(
mock.MagicMock(), 'TestRegion',
ignore_proxy_for_object_store=True)]:
self.assertEqual(proxy_env['HTTP_PROXY'],
'http://squid.internal:3128')
self.assertEqual(proxy_env['http_proxy'],
'http://squid.internal:3128')
self.assertEqual(proxy_env['HTTPS_PROXY'],
'https://squid.internal:3128')
self.assertEqual(proxy_env['https_proxy'],
'https://squid.internal:3128')
no_proxy_set = set(['127.0.0.1', 'localhost', '::1', '192.0.2.42',
'192.0.2.43', '192.0.2.44', '192.0.2.45',
'192.0.2.47', '192.0.2.90'])
self.assertEqual(set(proxy_env['NO_PROXY'].split(',')),
no_proxy_set)
self.assertEqual(set(proxy_env['no_proxy'].split(',')),
no_proxy_set)
# Make sure that object-store endpoints are not included into
# NO_PROXY when this is explicitly being asked for. In this case
# the set of expected addresses in NO_PROXY should exclude 192.0.2.90.
proxy_env = gss.get_sstream_mirror_proxy_env(
mock.MagicMock(),
'TestRegion', ignore_proxy_for_object_store=False)
self.assertEqual(proxy_env['HTTP_PROXY'], 'http://squid.internal:3128')
self.assertEqual(proxy_env['http_proxy'], 'http://squid.internal:3128')
self.assertEqual(proxy_env['HTTPS_PROXY'],
'https://squid.internal:3128')
self.assertEqual(proxy_env['https_proxy'],
'https://squid.internal:3128')
no_proxy_set_no_obj = set(['127.0.0.1', 'localhost', '::1',
'192.0.2.42', '192.0.2.43', '192.0.2.44',
'192.0.2.45', '192.0.2.47'])
self.assertEqual(set(proxy_env['NO_PROXY'].split(',')),
no_proxy_set_no_obj)
self.assertEqual(set(proxy_env['no_proxy'].split(',')),
no_proxy_set_no_obj)
def no_juju_proxy_settings_side_effect():
return None
juju_proxy_settings.side_effect = no_juju_proxy_settings_side_effect
# Make sure that even if Juju does not have any proxy settings set,
# via the model, we are still adding endpoints to NO_PROXY for
# sstream-mirror-glance invocations because settings might be sourced
# from other files (see glance-simplestreams-sync.sh).
proxy_env = gss.get_sstream_mirror_proxy_env(
mock.MagicMock(),
'TestRegion', ignore_proxy_for_object_store=False)
no_proxy_set_no_obj = set(['192.0.2.42', '192.0.2.43', '192.0.2.44',
'192.0.2.45', '192.0.2.47'])
self.assertEqual(set(proxy_env['NO_PROXY'].split(',')),
no_proxy_set_no_obj)
self.assertEqual(set(proxy_env['no_proxy'].split(',')),
no_proxy_set_no_obj)
def test_get_service_endpoints(self):
def url_for_side_effect(service_type, endpoint_type, region_name):
return {
'TestRegion': {
'identity': {
'publicURL': 'https://10.5.2.42:443/swift/v1',
'internalURL': 'https://10.5.2.42:443/swift/v1',
'adminURL': 'https://10.5.2.42:443/swift/v1',
},
'image': {
'publicURL': 'https://10.5.2.43:443/swift/v1',
'internalURL': 'https://10.5.2.43:443/swift/v1',
'adminURL': 'https://10.5.2.43:443/swift/v1',
},
'object-store': {
'publicURL': 'https://10.5.2.44:443/swift/v1',
'internalURL': 'https://10.5.2.44:443/swift/v1',
'adminURL': 'https://10.5.2.44:443/swift/v1',
},
}
}[region_name][service_type][endpoint_type]
ksc = mock.MagicMock()
ksc.service_catalog.url_for.side_effect = url_for_side_effect
self.assertEqual(
gss.get_service_endpoints(ksc, 'identity', 'TestRegion'), {
'publicURL': 'https://10.5.2.42:443/swift/v1',
'internalURL': 'https://10.5.2.42:443/swift/v1',
'adminURL': 'https://10.5.2.42:443/swift/v1',
}
)
self.assertEqual(
gss.get_service_endpoints(ksc, 'image', 'TestRegion'), {
'publicURL': 'https://10.5.2.43:443/swift/v1',
'internalURL': 'https://10.5.2.43:443/swift/v1',
'adminURL': 'https://10.5.2.43:443/swift/v1',
}
)
self.assertEqual(
gss.get_service_endpoints(ksc, 'object-store', 'TestRegion'), {
'publicURL': 'https://10.5.2.44:443/swift/v1',
'internalURL': 'https://10.5.2.44:443/swift/v1',
'adminURL': 'https://10.5.2.44:443/swift/v1',
}
)
ksc.service_catalog.url_for.side_effect = mock.MagicMock(
side_effect=keystone_exceptions.EndpointException('foo'))
with self.assertRaises(keystone_exceptions.EndpointException):
gss.get_service_endpoints(ksc, 'test', 'TestRegion')