[ADD] DCOS-2198 add proxy support

disable warning

improve requests

use env variables

change http(s)proxy to env variables

cli/dcoscli/analytics.py

@@ -5,10 +5,9 @@ import uuid
 
 import dcoscli
 import docopt
-import requests
 import rollbar
 from concurrent.futures import ThreadPoolExecutor
-from dcos import util
+from dcos import http, util
 from dcoscli.constants import (ROLLBAR_SERVER_POST_KEY,
                                SEGMENT_IO_CLI_ERROR_EVENT,
                                SEGMENT_IO_CLI_EVENT, SEGMENT_IO_WRITE_KEY_DEV,
@@ -104,10 +103,10 @@ def _segment_request(path, data):
         SEGMENT_IO_WRITE_KEY_DEV
 
     try:
-        requests.post('{}/{}'.format(SEGMENT_URL, path),
-                      json=data,
-                      auth=HTTPBasicAuth(key, ''),
-                      timeout=1)
+        http.post('{}/{}'.format(SEGMENT_URL, path),
+                  json=data,
+                  auth=HTTPBasicAuth(key, ''),
+                  timeout=1)
     except Exception as e:
         logger.exception(e)
 

cli/dcoscli/package/main.py

@@ -51,7 +51,8 @@ import sys
 import dcoscli
 import docopt
 import pkg_resources
-from dcos import cmds, emitting, marathon, options, package, subcommand, util
+from dcos import (cmds, emitting, http, marathon, options, package, subcommand,
+                  util)
 from dcos.errors import DCOSException
 from dcoscli import tables
 
@@ -74,6 +75,7 @@ def _main():
     args = docopt.docopt(
         __doc__,
         version='dcos-package version {}'.format(dcoscli.version))
+    http.silence_requests_warnings()
 
     return cmds.execute(_cmds(), args)
 

cli/tests/data/analytics/dcos_reporting.toml

@@ -1,3 +1,3 @@
 [core]
-email = "test@mail.com"
 reporting = true
+email = "test@mail.com"

cli/tests/integrations/test_analytics.py

@@ -2,9 +2,8 @@ import os
 from functools import wraps
 
 import dcoscli.analytics
-import requests
 import rollbar
-from dcos import constants, util
+from dcos import constants, http, util
 from dcoscli.analytics import _base_properties
 from dcoscli.config.main import main as config_main
 from dcoscli.constants import (ROLLBAR_SERVER_POST_KEY,
@@ -26,7 +25,7 @@ def _mock(fn):
     def wrapper(*args, **kwargs):
         with patch('rollbar.init'), \
                 patch('rollbar.report_message'), \
-                patch('requests.post'), \
+                patch('dcos.http.post'), \
                 patch('dcoscli.analytics.session_id'):
 
             dcoscli.analytics.session_id = ANON_ID
@@ -49,7 +48,7 @@ def test_production_setting_true():
     with patch('sys.argv', args), patch.dict(os.environ, env):
         assert main() == 0
 
-        _, kwargs = requests.post.call_args_list[0]
+        _, kwargs = http.post.call_args_list[0]
         assert kwargs['auth'].username == SEGMENT_IO_WRITE_KEY_PROD
 
         rollbar.init.assert_called_with(ROLLBAR_SERVER_POST_KEY, 'prod')
@@ -69,7 +68,7 @@ def test_production_setting_false():
     with patch('sys.argv', args), patch.dict(os.environ, env):
         assert main() == 0
 
-        _, kwargs = requests.post.call_args_list[0]
+        _, kwargs = http.post.call_args_list[0]
         assert kwargs['auth'].username == SEGMENT_IO_WRITE_KEY_DEV
 
         rollbar.init.assert_called_with(ROLLBAR_SERVER_POST_KEY, 'dev')
@@ -87,7 +86,7 @@ def test_config_set():
         assert config_main() == 0
 
         # segment.io
-        assert mock_called_some_args(requests.post,
+        assert mock_called_some_args(http.post,
                                      '{}/identify'.format(SEGMENT_URL),
                                      json={'userId': 'test@mail.com'},
                                      timeout=1)
@@ -110,7 +109,7 @@ def test_no_exc():
         data = {'userId': USER_ID,
                 'event': SEGMENT_IO_CLI_EVENT,
                 'properties': _base_properties()}
-        assert mock_called_some_args(requests.post,
+        assert mock_called_some_args(http.post,
                                      '{}/track'.format(SEGMENT_URL),
                                      json=data,
                                      timeout=1)
@@ -143,7 +142,7 @@ def test_exc():
                 'event': SEGMENT_IO_CLI_ERROR_EVENT,
                 'properties': props}
 
-        assert mock_called_some_args(requests.post,
+        assert mock_called_some_args(http.post,
                                      '{}/track'.format(SEGMENT_URL),
                                      json=data,
                                      timeout=1)
@@ -171,7 +170,7 @@ def test_config_reporting_false():
         assert main() == 1
 
         assert rollbar.report_message.call_count == 0
-        assert requests.post.call_count == 0
+        assert http.post.call_count == 0
 
 
 def _env_reporting():

dcos/http.py

@@ -83,7 +83,8 @@ def request(method,
             request.headers)
 
         with requests.Session() as session:
-            response = session.send(request.prepare(), timeout=timeout)
+            response = session.send(request.prepare(), timeout=timeout,
+                                    proxies=util.get_proxy_dict_from_env())
     except Exception as ex:
         raise to_exception(ex)
 

dcos/package.py

@@ -15,7 +15,8 @@ import git
 import portalocker
 import pystache
 import six
-from dcos import constants, emitting, errors, marathon, mesos, subcommand, util
+from dcos import (constants, emitting, errors, http, marathon, mesos,
+                  subcommand, util)
 from dcos.errors import DCOSException
 
 from six.moves import urllib
@@ -845,9 +846,14 @@ class HttpSource(Source):
             with util.tempdir() as tmp_dir:
 
                 tmp_file = os.path.join(tmp_dir, 'packages.zip')
-
                 # Download the zip file.
-                urllib.request.urlretrieve(self.url, tmp_file)
+                req = http.get(self.url)
+                if req.status_code == 200:
+                    with open(tmp_file, 'wb') as f:
+                        for chunk in req.iter_content(1024):
+                            f.write(chunk)
+                else:
+                    raise Exception
 
                 # Unzip the downloaded file.
                 packages_zip = zipfile.ZipFile(tmp_file, 'r')

dcos/util.py

@@ -547,3 +547,18 @@ def io_exception(path, errno):
 
 
 logger = get_logger(__name__)
+
+
+def get_proxy_dict_from_env():
+    """ Returns dict with proxy parameters
+
+    :returns: Dict with proxy parameters
+    :rtype: dict
+    """
+
+    proxies = dict()
+
+    for name, value in os.environ.items():
+        if value and (name == 'http_proxy' or name == 'https_proxy'):
+            proxies[name] = value
+    return proxies