x/cloud-init
Code Issues Proposed changes

Continue working on integrating requests.

Browse Source
This commit is contained in:
harlowja
2013-02-19 22:51:49 -08:00
parent 925f2b40f5
commit 2ad41a09a1
7 changed files with 129 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cloudinit/config/cc_phone_home.py
View File

@@ -19,7 +19,6 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from cloudinit import templater
from cloudinit import url_helper as uhelp
from cloudinit import util
from cloudinit.settings import PER_INSTANCE
@@ -112,7 +111,8 @@ def handle(name, cfg, cloud, log, args):
}
url = templater.render_string(url, url_params)
try:
uhelp.readurl(url, data=real_submit_keys, retries=tries, sec_between=3)
util.read_file_or_url(url, data=real_submit_keys,
retries=tries, sec_between=3)
except:
util.logexc(log, ("Failed to post phone home data to"
" %s in %s tries"), url, tries)

24
cloudinit/ec2_utils.py
View File

@@ -22,7 +22,6 @@ import json
import urllib
from cloudinit import log as logging
from cloudinit import url_helper as uh
from cloudinit import util
LOG = logging.getLogger(__name__)
@@ -40,11 +39,10 @@ def combine_url(base, add_on):
# See: http://bit.ly/TyoUQs
class MetadataMaterializer(object):
def __init__(self, blob, base_url, **fetch_settings):
def __init__(self, blob, base_url):
self._blob = blob
self._md = None
self._base_url = base_url
self._fetch_settings = fetch_settings
def _parse(self, blob):
leaves = {}
@@ -90,8 +88,8 @@ class MetadataMaterializer(object):
self._md = self._materialize(self._blob, self._base_url)
return self._md
def _fetch_url(self, url, **opts):
response = uh.readurl(url, **opts)
def _fetch_url(self, url):
response = util.read_file_or_url(url)
return str(response)
def _decode_leaf_blob(self, blob):
@@ -115,12 +113,12 @@ class MetadataMaterializer(object):
child_url = combine_url(base_url, c)
if not child_url.endswith("/"):
child_url += "/"
child_blob = self._fetch_url(child_url, **self._fetch_settings)
child_blob = self._fetch_url(child_url)
child_contents[c] = self._materialize(child_blob, child_url)
leaf_contents = {}
for (field, resource) in leaves.items():
leaf_url = combine_url(base_url, resource)
leaf_blob = self._fetch_url(leaf_url, **self._fetch_settings)
leaf_blob = self._fetch_url(leaf_url)
leaf_contents[field] = self._decode_leaf_blob(leaf_blob)
joined = {}
joined.update(child_contents)
@@ -136,23 +134,19 @@ def get_instance_userdata(url, version='latest', ssl_details=None):
ud_url = combine_url(url, version)
ud_url = combine_url(ud_url, 'user-data')
try:
response = uh.readurl(ud_url, timeout=5,
retries=10, ssl_details=ssl_details)
response = util.read_file_or_url(ud_url)
return str(response)
except Exception:
util.logexc(LOG, "Failed fetching userdata from url %s", ud_url)
return None
def get_instance_metadata(url, version='latest', ssl_details=None):
def get_instance_metadata(url, version='latest'):
md_url = combine_url(url, version)
md_url = combine_url(md_url, 'meta-data')
try:
response = uh.readurl(md_url, timeout=5,
retries=10, ssl_details=ssl_details)
materializer = MetadataMaterializer(str(response), md_url,
timeout=5, retries=10,
ssl_details=ssl_details)
response = util.read_file_or_url(md_url)
materializer = MetadataMaterializer(str(response), md_url)
return materializer.materialize()
except Exception:
util.logexc(LOG, "Failed fetching metadata from url %s", md_url)

2
cloudinit/sources/DataSourceEc2.py
View File

@@ -137,7 +137,7 @@ class DataSourceEc2(sources.DataSource):
start_time = time.time()
url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
timeout=timeout, status_cb=LOG.warn)
timeout=timeout, status_cb=LOG.warn)
if url:
LOG.debug("Using metadata source: '%s'", url2base[url])

16
cloudinit/sources/DataSourceMAAS.py
View File

@@ -25,9 +25,11 @@ import os
import time
import urllib2
import requests
from cloudinit import log as logging
from cloudinit import sources
from cloudinit import url_helper as uhelp
from cloudinit import url_helper
from cloudinit import util
LOG = logging.getLogger(__name__)
@@ -191,8 +193,8 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None,
version=MD_VERSION):
"""
Read the maas datasource at seed_url.
header_cb is a method that should return a headers dictionary that will
be given to urllib2.Request()
- header_cb is a method that should return a headers dictionary for
a given url
Expected format of seed_url is are the following files:
* <seed_url>/<version>/meta-data/instance-id
@@ -220,13 +222,13 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None,
else:
headers = {}
try:
resp = uhelp.readurl(url, headers=headers, timeout=timeout)
if resp.ok():
resp = util.read_file_or_url(url, headers=headers, timeout=timeout)
if resp.ok:
md[name] = str(resp)
else:
LOG.warn(("Fetching from %s resulted in"
" an invalid http code %s"), url, resp.code)
except urllib2.HTTPError as e:
" an invalid http code %s"), url, resp.status_code)
except url_helper.UrlError as e:
if e.code != 404:
raise
return check_seed_contents(md, seed_url)

50
cloudinit/url_helper.py
View File

@@ -58,6 +58,44 @@ def _cleanurl(url):
return urlunparse(parsed_url)
class UrlResponse(object):
def __init__(self, response):
self._response = response
@property
def contents(self):
return self._response.content
@property
def url(self):
return self._response.url
@property
def ok(self):
return self._response.ok
@property
def headers(self):
return self._response.headers
@property
def code(self):
return self._response.status_code
def __str__(self):
return self.contents
class UrlError(IOError):
def __init__(self, cause):
IOError.__init__(self, str(cause))
self.cause = cause
if isinstance(cause, exceptions.HTTPError) and cause.response:
self.code = cause.response.status_code
else:
self.code = None
def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
headers=None, ssl_details=None, check_status=True,
allow_redirects=False):
@@ -76,6 +114,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
if 'cert_file' in ssl_details and 'key_file' in ssl_details:
req_args['cert'] = [ssl_details['cert_file'],
ssl_details['key_file']]
elif 'cert_file' in ssl_details:
req_args['cert'] = str(ssl_details['cert_file'])
req_args['allow_redirects'] = allow_redirects
req_args['method'] = 'GET'
@@ -126,13 +166,9 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
# Doesn't seem like we can make it use a different
# subclass for responses, so add our own backward-compat
# attrs
if not hasattr(r, 'code'):
setattr(r, 'code', status)
if not hasattr(r, 'contents'):
setattr(r, 'contents', contents)
return r
return UrlResponse(r)
except exceptions.RequestException as e:
excps.append(e)
excps.append(UrlError(e))
if i + 1 < manual_tries and sec_between > 0:
LOG.debug("Please wait %s seconds while we wait to try again",
sec_between)
@@ -213,7 +249,7 @@ def wait_for_url(urls, max_wait=None, timeout=None,
e = ValueError(reason)
else:
return url
except exceptions.RequestException as e:
except UrlError as e:
reason = "request error [%s]" % e
except Exception as e:
reason = "unexpected error [%s]" % e

7
cloudinit/user_data.py
View File

@@ -29,7 +29,6 @@ from email.mime.text import MIMEText
from cloudinit import handlers
from cloudinit import log as logging
from cloudinit import url_helper
from cloudinit import util
LOG = logging.getLogger(__name__)
@@ -173,10 +172,10 @@ class UserDataProcessor(object):
if include_once_on and os.path.isfile(include_once_fn):
content = util.load_file(include_once_fn)
else:
resp = url_helper.readurl(include_url)
if include_once_on and resp.ok():
resp = util.read_file_or_url(include_url)
if include_once_on and resp.ok:
util.write_file(include_once_fn, str(resp), mode=0600)
if resp.ok():
if resp.ok:
content = str(resp)
else:
LOG.warn(("Fetching from %s resulted in"

71
cloudinit/util.py
View File

@@ -51,7 +51,7 @@ import yaml
from cloudinit import importer
from cloudinit import log as logging
from cloudinit import safeyaml
from cloudinit import url_helper as uhelp
from cloudinit import url_helper
from cloudinit.settings import (CFG_BUILTIN)
@@ -69,6 +69,18 @@ FN_ALLOWED = ('_-.()' + string.digits + string.ascii_letters)
CONTAINER_TESTS = ['running-in-container', 'lxc-is-container']
class FileResponse(object):
def __init__(self, path, contents):
self.code = 200
self.headers = {}
self.contents = contents
self.ok = True
self.url = path
def __str__(self):
return self.contents
class ProcessExecutionError(IOError):
MESSAGE_TMPL = ('%(description)s\n'
@@ -628,12 +640,53 @@ def read_optional_seed(fill, base="", ext="", timeout=5):
raise
def read_file_or_url(url, timeout=5, retries=10, file_retries=0):
def fetch_ssl_details(paths=None):
ssl_details = {}
# Lookup in these locations for ssl key/cert files
ssl_cert_paths = [
'/var/lib/cloud/data/ssl',
'/var/lib/cloud/instance/data/ssl',
]
if paths:
ssl_cert_paths.extend([
os.path.join(paths.get_ipath_cur('data'), 'ssl'),
os.path.join(paths.get_cpath('data'), 'ssl'),
])
ssl_cert_paths = uniq_merge(ssl_cert_paths)
ssl_cert_paths = [d for d in ssl_cert_paths if d and os.path.isdir(d)]
cert_file = None
for d in ssl_cert_paths:
if os.path.isfile(os.path.join(d, 'cert.pem')):
cert_file = os.path.join(d, 'cert.pem')
break
key_file = None
for d in ssl_cert_paths:
if os.path.isfile(os.path.join(d, 'key.pem')):
key_file = os.path.join(d, 'key.pem')
break
if cert_file and key_file:
ssl_details['cert_file'] = cert_file
ssl_details['key_file'] = key_file
elif cert_file:
ssl_details['cert_file'] = cert_file
return ssl_details
def read_file_or_url(url, timeout=5, retries=10,
headers=None, data=None, sec_between=1, paths=None):
if url.startswith("/"):
url = "file://%s" % url
if url.startswith("file://"):
retries = file_retries
return uhelp.readurl(url, timeout=timeout, retries=retries)
if url.lower().startswith("file://"):
file_path = url[len("file://"):]
return FileResponse(file_path, contents=load_file(file_path))
else:
return url_helper.readurl(url,
timeout=timeout,
retries=retries,
headers=headers,
data=data,
sec_between=sec_between,
ssl_details=fetch_ssl_details(paths))
def load_yaml(blob, default=None, allowed=(dict,)):
@@ -675,13 +728,13 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0):
md_resp = read_file_or_url(md_url, timeout, retries, file_retries)
md = None
if md_resp.ok():
if md_resp.ok:
md_str = str(md_resp)
md = load_yaml(md_str, default={})
ud_resp = read_file_or_url(ud_url, timeout, retries, file_retries)
ud = None
if ud_resp.ok():
if ud_resp.ok:
ud_str = str(ud_resp)
ud = ud_str
@@ -850,8 +903,8 @@ def get_cmdline_url(names=('cloud-config-url', 'url'),
if not url:
return (None, None, None)
resp = uhelp.readurl(url)
if resp.contents.startswith(starts) and resp.ok():
resp = read_file_or_url(url)
if resp.contents.startswith(starts) and resp.ok:
return (key, url, str(resp))
return (key, url, None)