DataSourceMAAS: if a oauth request fails due to 403 try updating local time

In the event of a 403 (Unauthorized) in oauth, try set a 'oauth_clockskew'
variable.  In future headers, use a time created by 'time.time() +
self.oauth_clockskew'.  The idea here is that if the local time is bad (or even
if the server time is bad) we will essentially use something that should be
similar to the remote clock.

This fixes LP: #978127.

cloudinit/sources/DataSourceMAAS.py

@@ -18,6 +18,7 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from email.utils import parsedate
 import errno
 import oauth.oauth as oauth
 import os
@@ -46,6 +47,7 @@ class DataSourceMAAS(sources.DataSource):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
         self.base_url = None
         self.seed_dir = os.path.join(paths.seed_dir, 'maas')
+        self.oauth_clockskew = None
 
     def __str__(self):
         return "%s [%s]" % (util.obj_name(self), self.base_url)
@@ -95,11 +97,17 @@ class DataSourceMAAS(sources.DataSource):
                 return {}
 
         consumer_secret = mcfg.get('consumer_secret', "")
+
+        timestamp = None
+        if self.oauth_clockskew:
+            timestamp = int(time.time()) + self.oauth_clockskew
+
         return oauth_headers(url=url,
                              consumer_key=mcfg['consumer_key'],
                              token_key=mcfg['token_key'],
                              token_secret=mcfg['token_secret'],
-                             consumer_secret=consumer_secret)
+                             consumer_secret=consumer_secret,
+                             timestamp=timestamp)
 
     def wait_for_metadata_service(self, url):
         mcfg = self.ds_cfg
@@ -124,7 +132,7 @@ class DataSourceMAAS(sources.DataSource):
         check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION)
         urls = [check_url]
         url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
-                                 timeout=timeout, status_cb=LOG.warn,
+                                 timeout=timeout, exception_cb=self._except_cb,
                                  headers_cb=self.md_headers)
 
         if url:
@@ -135,6 +143,26 @@ class DataSourceMAAS(sources.DataSource):
 
         return bool(url)
 
+    def _except_cb(self, msg, exception):
+        if not (isinstance(exception, urllib2.HTTPError) and
+                exception.code == 403):
+            return
+        if 'date' not in exception.headers:
+            LOG.warn("date field not in 403 headers")
+            return
+
+        date = exception.headers['date']
+
+        try:
+            ret_time = time.mktime(parsedate(date))
+        except:
+            LOG.warn("failed to convert datetime '%s'")
+            return
+
+        self.oauth_clockskew = int(ret_time - time.time())
+        LOG.warn("set oauth clockskew to %d" % self.oauth_clockskew)
+        return
+
 
 def read_maas_seed_dir(seed_d):
     """
@@ -229,13 +257,20 @@ def check_seed_contents(content, seed):
     return (userdata, md)
 
 
-def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret):
+def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret,
+                  timestamp=None):
     consumer = oauth.OAuthConsumer(consumer_key, consumer_secret)
     token = oauth.OAuthToken(token_key, token_secret)
+
+    if timestamp is None:
+        ts = int(time.time())
+    else:
+        ts = timestamp
+
     params = {
         'oauth_version': "1.0",
         'oauth_nonce': oauth.generate_nonce(),
-        'oauth_timestamp': int(time.time()),
+        'oauth_timestamp': ts,
         'oauth_token': token.key,
         'oauth_consumer_key': consumer.key,
     }

cloudinit/url_helper.py

@@ -136,7 +136,8 @@ def readurl(url, data=None, timeout=None,
 
 
 def wait_for_url(urls, max_wait=None, timeout=None,
-                 status_cb=None, headers_cb=None, sleep_time=1):
+                 status_cb=None, headers_cb=None, sleep_time=1,
+                 exception_cb=None):
     """
     urls:      a list of urls to try
     max_wait:  roughly the maximum time to wait before giving up
@@ -146,6 +147,8 @@ def wait_for_url(urls, max_wait=None, timeout=None,
     status_cb: call method with string message when a url is not available
     headers_cb: call method with single argument of url to get headers
                 for request.
+    exception_cb: call method with 2 arguments 'msg' (per status_cb) and
+                  'exception', the exception that occurred.
 
     the idea of this routine is to wait for the EC2 metdata service to
     come up.  On both Eucalyptus and EC2 we have seen the case where
@@ -164,7 +167,7 @@ def wait_for_url(urls, max_wait=None, timeout=None,
     """
     start_time = time.time()
 
-    def log_status_cb(msg):
+    def log_status_cb(msg, exc=None):
         LOG.debug(msg)
 
     if status_cb is None:
@@ -196,8 +199,10 @@ def wait_for_url(urls, max_wait=None, timeout=None,
                 resp = readurl(url, headers=headers, timeout=timeout)
                 if not resp.contents:
                     reason = "empty response [%s]" % (resp.code)
+                    e = ValueError(reason)
                 elif not resp.ok():
                     reason = "bad status code [%s]" % (resp.code)
+                    e = ValueError(reason)
                 else:
                     return url
             except urllib2.HTTPError as e:
@@ -214,6 +219,8 @@ def wait_for_url(urls, max_wait=None, timeout=None,
                                                              time_taken,
                                                              max_wait, reason)
             status_cb(status_msg)
+            if exception_cb:
+                exception_cb(msg=status_msg, exception=e)
 
         if timeup(max_wait, start_time):
             break