Implements fetching from HTTP sources.

dcos/api/package.py

@@ -4,22 +4,28 @@ import hashlib
 import json
 import os
 import shutil
+import stat
 import subprocess
+import zipfile
 
 import git
 import portalocker
 import pystache
-from dcos.api import errors, util
+from dcos.api import emitting, errors, util
 
 try:
     # Python 2
     from urlparse import urlparse
+    from urllib import urlretrieve
 except ImportError:
     # Python 3
     from urllib.parse import urlparse
+    from urllib.request import urlretrieve
 
 logger = util.get_logger(__name__)
 
+emitter = emitting.FlatEmitter()
+
 
 PACKAGE_NAME_KEY = 'DCOS_PACKAGE_NAME'
 PACKAGE_VERSION_KEY = 'DCOS_PACKAGE_VERSION'
@@ -338,7 +344,7 @@ def update_sources(config):
 
         for source in sources:
 
-            logger.info("Updating source [%s]", source)
+            emitter.publish('Updating source [{}]'.format(source))
 
             # create a temporary staging directory
             with util.tempdir() as tmp_dir:
@@ -462,8 +468,7 @@ class FileSource(Source):
             shutil.copytree(source_dir, target_dir)
             return None
         except OSError:
-            return Error('Could not copy [{}] to [{}]'.format(source_dir,
-                                                              target_dir))
+            return Error('Unable to fetch packages from [{}]'.format(self.url))
 
 
 class HttpSource(Source):
@@ -494,7 +499,47 @@ class HttpSource(Source):
         :rtype: Error
         """
 
-        raise NotImplementedError
+        try:
+            with util.tempdir() as tmp_dir:
+
+                tmp_file = os.path.join(tmp_dir, 'packages.zip')
+
+                # Download the zip file.
+                urlretrieve(self.url, tmp_file)
+
+                # Unzip the downloaded file.
+                packages_zip = zipfile.ZipFile(tmp_file, 'r')
+                packages_zip.extractall(tmp_dir)
+
+                # Move the enclosing directory to the target directory
+                enclosing_dirs = [item
+                                  for item in os.listdir(tmp_dir)
+                                  if os.path.isdir(
+                                      os.path.join(tmp_dir, item))]
+
+                # There should only be one directory present after extracting.
+                assert(len(enclosing_dirs) is 1)
+
+                enclosing_dir = os.path.join(tmp_dir, enclosing_dirs[0])
+
+                shutil.copytree(enclosing_dir, target_dir)
+
+                # Set appropriate file permissions on the scripts.
+                x_mode = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
+                          stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP)
+
+                scripts_dir = os.path.join(target_dir, 'scripts')
+                scripts = os.listdir(scripts_dir)
+
+                for script in scripts:
+                    script_path = os.path.join(scripts_dir, script)
+                    if os.path.isfile(script_path):
+                        os.chmod(script_path, x_mode)
+
+                return None
+
+        except Exception:
+            return Error('Unable to fetch packages from [{}]'.format(self.url))
 
 
 class GitSource(Source):
@@ -546,8 +591,7 @@ PATH = {}""".format(os.environ['PATH']))
             return None
 
         except git.exc.GitCommandError:
-            return Error("Unable to clone [{}] to [{}]".format(self.url,
-                                                               target_dir))
+            return Error("Unable to fetch packages from [{}]".format(self.url))
 
 
 class Error(errors.Error):

integrations/cli/test_config.py

@@ -42,7 +42,8 @@ def test_list_property():
     assert stdout == b"""marathon.host=localhost
 marathon.port=8080
 package.cache=tmp/cache
-package.sources=['git://github.com/mesosphere/universe.git']
+package.sources=['git://github.com/mesosphere/universe.git', \
+'https://github.com/mesosphere/universe/archive/master.zip']
 """
     assert stderr == b''
 

integrations/cli/test_package.py

@@ -61,6 +61,8 @@ def test_sources_list():
     assert returncode == 0
     assert stdout == b"""c3f1a0df1d2068e6b11d40224f5e500d3183a97e \
 git://github.com/mesosphere/universe.git
+f4ba0923d14eb75c1c0afca61c2adf9b2b355bd5 \
+https://github.com/mesosphere/universe/archive/master.zip
 """
     assert stderr == b''
 
@@ -69,8 +71,9 @@ def test_update():
     returncode, stdout, stderr = exec_command(['dcos', 'package', 'update'])
 
     assert returncode == 0
-    assert stdout.startswith(b'Validating package definitions...')
-    assert stdout.endswith(b'OK\n')
+    assert b'source' in stdout
+    assert b'Validating package definitions...' in stdout
+    assert b'OK' in stdout
     assert stderr == b''
 
 

tests/data/Dcos.toml

@@ -1,6 +1,9 @@
 [package]
-sources = [ "git://github.com/mesosphere/universe.git",]
 cache = "tmp/cache"
+sources = [
+  "git://github.com/mesosphere/universe.git",
+  "https://github.com/mesosphere/universe/archive/master.zip"
+]
 [marathon]
 host = "localhost"
 port = 8080