drop requirement on boto for its boto.utils.get_instance_metadata()
We had a requirement on boto only to use boto.utils.get_instance_metadata(). That had actually caused some pain in the past. This removes a Requires and also one that wasn't python3.
This commit is contained in:
Scott Moser
commit
6b92d3496b
|
|
@ -14,6 +14,7 @@
|
|||
- drop support for resizing partitions with parted entirely (LP: #1212492).
|
||||
This was broken as it was anyway.
|
||||
- add support for vendordata.
|
||||
- drop dependency on boto for crawling ec2 metadata service.
|
||||
0.7.4:
|
||||
- fix issue mounting 'ephemeral0' if ephemeral0 was an alias for a
|
||||
partitioned block device with target filesystem on ephemeral0.1.
|
||||
|
|
|
|||
6
Requires
6
Requires
|
|
@ -29,8 +29,8 @@ argparse
|
|||
# Requests handles ssl correctly!
|
||||
requests
|
||||
|
||||
# Boto for ec2
|
||||
boto
|
||||
|
||||
# For patching pieces of cloud-config together
|
||||
jsonpatch
|
||||
|
||||
# For http testing (only needed for testing)
|
||||
httpretty>=0.7.1
|
||||
|
|
|
|||
|
|
@ -16,48 +16,163 @@
|
|||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import boto.utils as boto_utils
|
||||
from urlparse import (urlparse, urlunparse)
|
||||
|
||||
# Versions of boto >= 2.6.0 (and possibly 2.5.2)
|
||||
# try to lazily load the metadata backing, which
|
||||
# doesn't work so well in cloud-init especially
|
||||
# since the metadata is serialized and actions are
|
||||
# performed where the metadata server may be blocked
|
||||
# (thus the datasource will start failing) resulting
|
||||
# in url exceptions when fields that do exist (or
|
||||
# would have existed) do not exist due to the blocking
|
||||
# that occurred.
|
||||
import functools
|
||||
import json
|
||||
import urllib
|
||||
|
||||
# TODO(harlowja): https://github.com/boto/boto/issues/1401
|
||||
# When boto finally moves to using requests, we should be able
|
||||
# to provide it ssl details, it does not yet, so we can't provide them...
|
||||
from cloudinit import log as logging
|
||||
from cloudinit import util
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _unlazy_dict(mp):
|
||||
if not isinstance(mp, (dict)):
|
||||
return mp
|
||||
# Walk over the keys/values which
|
||||
# forces boto to unlazy itself and
|
||||
# has no effect on dictionaries that
|
||||
# already have there items.
|
||||
for (_k, v) in mp.items():
|
||||
_unlazy_dict(v)
|
||||
return mp
|
||||
def maybe_json_object(text):
|
||||
if not text:
|
||||
return False
|
||||
text = text.strip()
|
||||
if text.startswith("{") and text.endswith("}"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_instance_userdata(api_version, metadata_address):
|
||||
# Note: boto.utils.get_instance_metadata returns '' for empty string
|
||||
# so the change from non-true to '' is not specifically necessary, but
|
||||
# this way cloud-init will get consistent behavior even if boto changed
|
||||
# in the future to return a None on "no user-data provided".
|
||||
ud = boto_utils.get_instance_userdata(api_version, None, metadata_address)
|
||||
if not ud:
|
||||
ud = ''
|
||||
return ud
|
||||
def combine_url(base, add_on):
|
||||
base_parsed = list(urlparse(base))
|
||||
path = base_parsed[2]
|
||||
if path and not path.endswith("/"):
|
||||
path += "/"
|
||||
path += urllib.quote(str(add_on), safe="/:")
|
||||
base_parsed[2] = path
|
||||
return urlunparse(base_parsed)
|
||||
|
||||
|
||||
def get_instance_metadata(api_version, metadata_address):
|
||||
metadata = boto_utils.get_instance_metadata(api_version, metadata_address)
|
||||
if not isinstance(metadata, (dict)):
|
||||
metadata = {}
|
||||
return _unlazy_dict(metadata)
|
||||
# See: http://bit.ly/TyoUQs
|
||||
#
|
||||
# Since boto metadata reader uses the old urllib which does not
|
||||
# support ssl, we need to ahead and create our own reader which
|
||||
# works the same as the boto one (for now).
|
||||
class MetadataMaterializer(object):
|
||||
def __init__(self, blob, base_url, caller):
|
||||
self._blob = blob
|
||||
self._md = None
|
||||
self._base_url = base_url
|
||||
self._caller = caller
|
||||
|
||||
def _parse(self, blob):
|
||||
leaves = {}
|
||||
children = []
|
||||
if not blob:
|
||||
return (leaves, children)
|
||||
|
||||
def has_children(item):
|
||||
if item.endswith("/"):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def get_name(item):
|
||||
if item.endswith("/"):
|
||||
return item.rstrip("/")
|
||||
return item
|
||||
|
||||
for field in blob.splitlines():
|
||||
field = field.strip()
|
||||
field_name = get_name(field)
|
||||
if not field or not field_name:
|
||||
continue
|
||||
if has_children(field):
|
||||
if field_name not in children:
|
||||
children.append(field_name)
|
||||
else:
|
||||
contents = field.split("=", 1)
|
||||
resource = field_name
|
||||
if len(contents) > 1:
|
||||
# What a PITA...
|
||||
(ident, sub_contents) = contents
|
||||
ident = util.safe_int(ident)
|
||||
if ident is not None:
|
||||
resource = "%s/openssh-key" % (ident)
|
||||
field_name = sub_contents
|
||||
leaves[field_name] = resource
|
||||
return (leaves, children)
|
||||
|
||||
def materialize(self):
|
||||
if self._md is not None:
|
||||
return self._md
|
||||
self._md = self._materialize(self._blob, self._base_url)
|
||||
return self._md
|
||||
|
||||
def _decode_leaf_blob(self, field, blob):
|
||||
if not blob:
|
||||
return blob
|
||||
if maybe_json_object(blob):
|
||||
try:
|
||||
# Assume it's json, unless it fails parsing...
|
||||
return json.loads(blob)
|
||||
except (ValueError, TypeError) as e:
|
||||
LOG.warn("Field %s looked like a json object, but it was"
|
||||
" not: %s", field, e)
|
||||
if blob.find("\n") != -1:
|
||||
return blob.splitlines()
|
||||
return blob
|
||||
|
||||
def _materialize(self, blob, base_url):
|
||||
(leaves, children) = self._parse(blob)
|
||||
child_contents = {}
|
||||
for c in children:
|
||||
child_url = combine_url(base_url, c)
|
||||
if not child_url.endswith("/"):
|
||||
child_url += "/"
|
||||
child_blob = str(self._caller(child_url))
|
||||
child_contents[c] = self._materialize(child_blob, child_url)
|
||||
leaf_contents = {}
|
||||
for (field, resource) in leaves.items():
|
||||
leaf_url = combine_url(base_url, resource)
|
||||
leaf_blob = str(self._caller(leaf_url))
|
||||
leaf_contents[field] = self._decode_leaf_blob(field, leaf_blob)
|
||||
joined = {}
|
||||
joined.update(child_contents)
|
||||
for field in leaf_contents.keys():
|
||||
if field in joined:
|
||||
LOG.warn("Duplicate key found in results from %s", base_url)
|
||||
else:
|
||||
joined[field] = leaf_contents[field]
|
||||
return joined
|
||||
|
||||
|
||||
def get_instance_userdata(api_version='latest',
|
||||
metadata_address='http://169.254.169.254',
|
||||
ssl_details=None, timeout=5, retries=5):
|
||||
ud_url = combine_url(metadata_address, api_version)
|
||||
ud_url = combine_url(ud_url, 'user-data')
|
||||
try:
|
||||
response = util.read_file_or_url(ud_url,
|
||||
ssl_details=ssl_details,
|
||||
timeout=timeout,
|
||||
retries=retries)
|
||||
return str(response)
|
||||
except Exception:
|
||||
util.logexc(LOG, "Failed fetching userdata from url %s", ud_url)
|
||||
return ''
|
||||
|
||||
|
||||
def get_instance_metadata(api_version='latest',
|
||||
metadata_address='http://169.254.169.254',
|
||||
ssl_details=None, timeout=5, retries=5):
|
||||
md_url = combine_url(metadata_address, api_version)
|
||||
md_url = combine_url(md_url, 'meta-data')
|
||||
caller = functools.partial(util.read_file_or_url,
|
||||
ssl_details=ssl_details, timeout=timeout,
|
||||
retries=retries)
|
||||
|
||||
try:
|
||||
response = caller(md_url)
|
||||
materializer = MetadataMaterializer(str(response), md_url, caller)
|
||||
md = materializer.materialize()
|
||||
if not isinstance(md, (dict)):
|
||||
md = {}
|
||||
return md
|
||||
except Exception:
|
||||
util.logexc(LOG, "Failed fetching metadata from url %s", md_url)
|
||||
return {}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,130 @@
|
|||
from tests.unittests import helpers
|
||||
|
||||
from cloudinit import ec2_utils as eu
|
||||
|
||||
import httpretty as hp
|
||||
|
||||
|
||||
class TestEc2Util(helpers.TestCase):
|
||||
VERSION = 'latest'
|
||||
|
||||
@hp.activate
|
||||
def test_userdata_fetch(self):
|
||||
hp.register_uri(hp.GET,
|
||||
'http://169.254.169.254/%s/user-data' % (self.VERSION),
|
||||
body='stuff',
|
||||
status=200)
|
||||
userdata = eu.get_instance_userdata(self.VERSION)
|
||||
self.assertEquals('stuff', userdata)
|
||||
|
||||
@hp.activate
|
||||
def test_userdata_fetch_fail_not_found(self):
|
||||
hp.register_uri(hp.GET,
|
||||
'http://169.254.169.254/%s/user-data' % (self.VERSION),
|
||||
status=404)
|
||||
userdata = eu.get_instance_userdata(self.VERSION, retries=0)
|
||||
self.assertEquals('', userdata)
|
||||
|
||||
@hp.activate
|
||||
def test_userdata_fetch_fail_server_dead(self):
|
||||
hp.register_uri(hp.GET,
|
||||
'http://169.254.169.254/%s/user-data' % (self.VERSION),
|
||||
status=500)
|
||||
userdata = eu.get_instance_userdata(self.VERSION, retries=0)
|
||||
self.assertEquals('', userdata)
|
||||
|
||||
@hp.activate
|
||||
def test_metadata_fetch_no_keys(self):
|
||||
base_url = 'http://169.254.169.254/%s/meta-data' % (self.VERSION)
|
||||
hp.register_uri(hp.GET, base_url, status=200,
|
||||
body="\n".join(['hostname',
|
||||
'instance-id',
|
||||
'ami-launch-index']))
|
||||
hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'),
|
||||
status=200, body='ec2.fake.host.name.com')
|
||||
hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'),
|
||||
status=200, body='123')
|
||||
hp.register_uri(hp.GET, eu.combine_url(base_url, 'ami-launch-index'),
|
||||
status=200, body='1')
|
||||
md = eu.get_instance_metadata(self.VERSION, retries=0)
|
||||
self.assertEquals(md['hostname'], 'ec2.fake.host.name.com')
|
||||
self.assertEquals(md['instance-id'], '123')
|
||||
self.assertEquals(md['ami-launch-index'], '1')
|
||||
|
||||
@hp.activate
|
||||
def test_metadata_fetch_key(self):
|
||||
base_url = 'http://169.254.169.254/%s/meta-data' % (self.VERSION)
|
||||
hp.register_uri(hp.GET, base_url, status=200,
|
||||
body="\n".join(['hostname',
|
||||
'instance-id',
|
||||
'public-keys/']))
|
||||
hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'),
|
||||
status=200, body='ec2.fake.host.name.com')
|
||||
hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'),
|
||||
status=200, body='123')
|
||||
hp.register_uri(hp.GET, eu.combine_url(base_url, 'public-keys/'),
|
||||
status=200, body='0=my-public-key')
|
||||
hp.register_uri(hp.GET,
|
||||
eu.combine_url(base_url, 'public-keys/0/openssh-key'),
|
||||
status=200, body='ssh-rsa AAAA.....wZEf my-public-key')
|
||||
md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
|
||||
self.assertEquals(md['hostname'], 'ec2.fake.host.name.com')
|
||||
self.assertEquals(md['instance-id'], '123')
|
||||
self.assertEquals(1, len(md['public-keys']))
|
||||
|
||||
@hp.activate
|
||||
def test_metadata_fetch_with_2_keys(self):
|
||||
base_url = 'http://169.254.169.254/%s/meta-data' % (self.VERSION)
|
||||
hp.register_uri(hp.GET, base_url, status=200,
|
||||
body="\n".join(['hostname',
|
||||
'instance-id',
|
||||
'public-keys/']))
|
||||
hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'),
|
||||
status=200, body='ec2.fake.host.name.com')
|
||||
hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'),
|
||||
status=200, body='123')
|
||||
hp.register_uri(hp.GET, eu.combine_url(base_url, 'public-keys/'),
|
||||
status=200,
|
||||
body="\n".join(['0=my-public-key', '1=my-other-key']))
|
||||
hp.register_uri(hp.GET,
|
||||
eu.combine_url(base_url, 'public-keys/0/openssh-key'),
|
||||
status=200, body='ssh-rsa AAAA.....wZEf my-public-key')
|
||||
hp.register_uri(hp.GET,
|
||||
eu.combine_url(base_url, 'public-keys/1/openssh-key'),
|
||||
status=200, body='ssh-rsa AAAA.....wZEf my-other-key')
|
||||
md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
|
||||
self.assertEquals(md['hostname'], 'ec2.fake.host.name.com')
|
||||
self.assertEquals(md['instance-id'], '123')
|
||||
self.assertEquals(2, len(md['public-keys']))
|
||||
|
||||
@hp.activate
|
||||
def test_metadata_fetch_bdm(self):
|
||||
base_url = 'http://169.254.169.254/%s/meta-data' % (self.VERSION)
|
||||
hp.register_uri(hp.GET, base_url, status=200,
|
||||
body="\n".join(['hostname',
|
||||
'instance-id',
|
||||
'block-device-mapping/']))
|
||||
hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'),
|
||||
status=200, body='ec2.fake.host.name.com')
|
||||
hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'),
|
||||
status=200, body='123')
|
||||
hp.register_uri(hp.GET,
|
||||
eu.combine_url(base_url, 'block-device-mapping/'),
|
||||
status=200,
|
||||
body="\n".join(['ami', 'ephemeral0']))
|
||||
hp.register_uri(hp.GET,
|
||||
eu.combine_url(base_url, 'block-device-mapping/ami'),
|
||||
status=200,
|
||||
body="sdb")
|
||||
hp.register_uri(hp.GET,
|
||||
eu.combine_url(base_url,
|
||||
'block-device-mapping/ephemeral0'),
|
||||
status=200,
|
||||
body="sdc")
|
||||
md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
|
||||
self.assertEquals(md['hostname'], 'ec2.fake.host.name.com')
|
||||
self.assertEquals(md['instance-id'], '123')
|
||||
bdm = md['block-device-mapping']
|
||||
self.assertEquals(2, len(bdm))
|
||||
self.assertEquals(bdm['ami'], 'sdb')
|
||||
self.assertEquals(bdm['ephemeral0'], 'sdc')
|
||||
Loading…
Reference in New Issue