Merge pull request #218 from mesosphere/windows_integration_tests

[ADD] DCOS-1002  Create a TeamCity integrated test for Windows version of DCOS CLI

cli/tests/integrations/common.py

@@ -30,7 +30,9 @@ def exec_command(cmd, env=None, stdin=None):
         stderr=subprocess.PIPE,
         env=env)
 
-    stdout, stderr = process.communicate()
+    # This is needed to get rid of '\r' from Windows's lines endings.
+    stdout, stderr = [std_stream.replace(b'\r', b'')
+                      for std_stream in process.communicate()]
 
     # We should always print the stdout and stderr
     print('STDOUT: {}'.format(_truncate(stdout.decode('utf-8'))))

cli/tests/integrations/test_analytics.py

@@ -35,6 +35,64 @@ def _mock(fn):
     return wrapper
 
 
+@_mock
+def test_production_setting_true():
+    '''Test that env var DCOS_PRODUCTION as empty string sends exceptions
+    to the 'prod' environment.
+
+    '''
+
+    args = [util.which('dcos')]
+    env = _env_reporting()
+    env['DCOS_PRODUCTION'] = ''
+
+    with patch('sys.argv', args), patch.dict(os.environ, env):
+        assert main() == 0
+
+        _, kwargs = requests.post.call_args_list[0]
+        assert kwargs['auth'].username == SEGMENT_IO_WRITE_KEY_PROD
+
+        rollbar.init.assert_called_with(ROLLBAR_SERVER_POST_KEY, 'prod')
+
+
+@_mock
+def test_production_setting_false():
+    '''Test that env var DCOS_PRODUCTION=false sends exceptions to
+    the 'dev' environment.
+
+    '''
+
+    args = [util.which('dcos')]
+    env = _env_reporting()
+    env['DCOS_PRODUCTION'] = 'false'
+
+    with patch('sys.argv', args), patch.dict(os.environ, env):
+        assert main() == 0
+
+        _, kwargs = requests.post.call_args_list[0]
+        assert kwargs['auth'].username == SEGMENT_IO_WRITE_KEY_DEV
+
+        rollbar.init.assert_called_with(ROLLBAR_SERVER_POST_KEY, 'dev')
+
+
+@_mock
+def test_config_set():
+    '''Tests that a `dcos config set core.email <email>` makes a
+    segment.io identify call'''
+
+    args = [util.which('dcos'), 'config', 'set', 'core.email', 'test@mail.com']
+    env = _env_reporting()
+
+    with patch('sys.argv', args), patch.dict(os.environ, env):
+        assert config_main() == 0
+
+        # segment.io
+        assert mock_called_some_args(requests.post,
+                                     '{}/identify'.format(SEGMENT_URL),
+                                     json={'userId': 'test@mail.com'},
+                                     timeout=1)
+
+
 @_mock
 def test_no_exc():
     '''Tests that a command which does not raise an exception does not
@@ -115,64 +173,6 @@ def test_config_reporting_false():
         assert requests.post.call_count == 0
 
 
-@_mock
-def test_production_setting_true():
-    '''Test that env var DCOS_PRODUCTION as empty string sends exceptions
-    to the 'prod' environment.
-
-    '''
-
-    args = [util.which('dcos')]
-    env = _env_reporting()
-    env['DCOS_PRODUCTION'] = ''
-
-    with patch('sys.argv', args), patch.dict(os.environ, env):
-        assert main() == 0
-
-        _, kwargs = requests.post.call_args_list[0]
-        assert kwargs['auth'].username == SEGMENT_IO_WRITE_KEY_PROD
-
-        rollbar.init.assert_called_with(ROLLBAR_SERVER_POST_KEY, 'prod')
-
-
-@_mock
-def test_production_setting_false():
-    '''Test that env var DCOS_PRODUCTION=false sends exceptions to
-    the 'dev' environment.
-
-    '''
-
-    args = [util.which('dcos')]
-    env = _env_reporting()
-    env['DCOS_PRODUCTION'] = 'false'
-
-    with patch('sys.argv', args), patch.dict(os.environ, env):
-        assert main() == 0
-
-        _, kwargs = requests.post.call_args_list[0]
-        assert kwargs['auth'].username == SEGMENT_IO_WRITE_KEY_DEV
-
-        rollbar.init.assert_called_with(ROLLBAR_SERVER_POST_KEY, 'dev')
-
-
-@_mock
-def test_config_set():
-    '''Tests that a `dcos config set core.email <email>` makes a
-    segment.io identify call'''
-
-    args = [util.which('dcos'), 'config', 'set', 'core.email', 'test@mail.com']
-    env = _env_reporting()
-
-    with patch('sys.argv', args), patch.dict(os.environ, env):
-        assert config_main() == 0
-
-        # segment.io
-        assert mock_called_some_args(requests.post,
-                                     '{}/identify'.format(SEGMENT_URL),
-                                     json={'userId': 'test@mail.com'},
-                                     timeout=1)
-
-
 def _env_reporting():
     path = os.path.join('tests', 'data', 'analytics', 'dcos_reporting.toml')
     return {constants.DCOS_CONFIG_ENV: path}

cli/tests/integrations/test_config.py

@@ -12,20 +12,24 @@ from .common import assert_command, exec_command
 
 @pytest.fixture
 def env():
-    return {
+    r = os.environ.copy()
+    r.update({
         constants.PATH_ENV: os.environ[constants.PATH_ENV],
         constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "dcos.toml"),
         cli_constants.DCOS_PRODUCTION_ENV: 'false'
-    }
+    })
+    return r
 
 
 @pytest.fixture
 def missing_env():
-    return {
+    r = os.environ.copy()
+    r.update({
         constants.PATH_ENV: os.environ[constants.PATH_ENV],
         constants.DCOS_CONFIG_ENV:
             os.path.join("tests", "data", "missing_params_dcos.toml")
-    }
+    })
+    return r
 
 
 def test_help():

cli/tests/integrations/test_dcos.py

@@ -71,9 +71,11 @@ def test_version():
 
 
 def test_missing_dcos_config():
-    env = {
+    env = os.environ.copy()
+    del env['DCOS_CONFIG']
+    env.update({
         constants.PATH_ENV: os.environ[constants.PATH_ENV],
-    }
+    })
 
     stdout = (b"Environment variable 'DCOS_CONFIG' must be set "
               b"to the DCOS config file.\n")
@@ -85,10 +87,11 @@ def test_missing_dcos_config():
 
 
 def test_dcos_config_not_a_file():
-    env = {
+    env = os.environ.copy()
+    env.update({
         constants.PATH_ENV: os.environ[constants.PATH_ENV],
         'DCOS_CONFIG': 'missing/file',
-    }
+    })
 
     stdout = (b"Environment variable 'DCOS_CONFIG' maps to "
               b"'missing/file' and it is not a file.\n")

cli/tests/integrations/test_marathon.py

@@ -134,11 +134,13 @@ def test_about():
 
 @pytest.fixture
 def missing_env():
-    return {
+    env = os.environ.copy()
+    env.update({
         constants.PATH_ENV: os.environ[constants.PATH_ENV],
         constants.DCOS_CONFIG_ENV:
             os.path.join("tests", "data", "missing_marathon_params.toml")
-    }
+    })
+    return env
 
 
 def test_missing_config(missing_env):

cli/tox.win.ini

@@ -0,0 +1,41 @@
+[tox]
+envlist = py{27,34}-unit, py{27,34}-integration
+
+[testenv]
+setenv =
+  DCOS_CONFIG = {env:DCOS_CONFIG}
+  EXHIBITOR_URL = {env:EXHIBITOR_URL}
+deps =
+  teamcity-messages
+  pytest
+  pytest-cov
+  mock
+  pypiwin32
+  -e..
+
+[testenv:syntax]
+deps =
+  teamcity-messages
+  flake8
+  isort
+  ..
+
+commands =
+  flake8 --verbose {env:CI_FLAGS:} dcoscli tests setup.py
+  isort --recursive --check-only --diff --verbose dcoscli tests setup.py
+
+[testenv:py27-integration]
+commands =
+  py.test -vv {env:CI_FLAGS:} tests/integrations{posargs}
+
+[testenv:py34-integration]
+commands =
+  py.test -vv {env:CI_FLAGS:} tests/integrations{posargs}
+
+[testenv:py27-unit]
+commands =
+  py.test -vv {env:CI_FLAGS:} tests/unit{posargs}
+
+[testenv:py34-unit]
+commands =
+  py.test -vv {env:CI_FLAGS:} tests/unit{posargs}

cli/win_bin/clean.ps1

@@ -0,0 +1,5 @@
+$BaseDir = (Get-Location).Path
+Remove-Item $BASEDIR\env -Recurse -Force -erroraction 'silentlycontinue'
+Remove-Item $BASEDIR\dist -Recurse -Force -erroraction 'silentlycontinue'
+Remove-Item $BASEDIR\build -Recurse -Force -erroraction 'silentlycontinue'
+Get-ChildItem -Path $BaseDir -Filter *.pyc -Recurse | Remove-Item -Force

cli/win_bin/env.ps1

@@ -0,0 +1,22 @@
+$BaseDir = (Get-Location).Path
+if (-Not (Test-Path $BaseDir\env -PathType Container)) {
+    virtualenv -q $BASEDIR/env --prompt='(dcoscli) '
+    echo "Virtualenv created."
+
+    & $BASEDIR/env/Scripts/activate
+    echo "Virtualenv activated."
+
+    & $BASEDIR/env/Scripts/pip.exe install -r $BASEDIR/requirements.txt
+    & $BASEDIR/env/Scripts/pip.exe install -e $BASEDIR
+    echo "Requirements installed."
+}
+ElseIf ((Test-Path $BASEDIR/env/bin/activate )) {
+
+    & $BASEDIR/env/Scripts/activate
+    echo "Virtualenv activated."
+
+    & $BASEDIR/env/Scripts/pip install -r $BASEDIR/requirements.txt
+    & $BASEDIR/env/Scripts/pip install -e $BASEDIR
+    echo "Requirements installed."
+
+}

cli/win_bin/packages.ps1

@@ -0,0 +1,7 @@
+$BaseDir = (Get-Location).Path
+
+echo "Building wheel..."
+"$BASEDIR/env/bin/python" setup.py bdist_wheel
+
+echo "Building egg..."
+"$BASEDIR/env/bin/python" setup.py sdist

cli/win_bin/test.ps1

@@ -0,0 +1,8 @@
+$BaseDir = (Get-Location).Path
+
+cd $BASEDIR
+& $BASEDIR\env\Scripts\activate
+echo "Virtualenv activated."
+
+tox -c tox.win.ini
+exit $LastExitCode

dcos/package.py

@@ -708,7 +708,9 @@ def update_sources(config, validate=False):
                 target_dir = os.path.join(cache_dir, source.hash())
                 try:
                     if os.path.exists(target_dir):
-                        shutil.rmtree(target_dir, ignore_errors=False)
+                        shutil.rmtree(target_dir,
+                                      onerror=_rmtree_on_error,
+                                      ignore_errors=False)
                 except OSError:
                     err = Error(
                         'Could not remove directory [{}]'.format(target_dir))
@@ -926,7 +928,8 @@ PATH = {}""".format(os.environ[constants.PATH_ENV]))
                                 branch='master')
 
             # Remove .git directory to save space.
-            shutil.rmtree(os.path.join(target_dir, ".git"))
+            shutil.rmtree(os.path.join(target_dir, ".git"),
+                          onerror=_rmtree_on_error)
             return None
 
         except git.exc.GitCommandError:
@@ -934,6 +937,30 @@ PATH = {}""".format(os.environ[constants.PATH_ENV]))
                 'Unable to fetch packages from [{}]'.format(self.url))
 
 
+def _rmtree_on_error(func, path, exc_info):
+    """Error handler for ``shutil.rmtree``.
+    If the error is due to an access error (read only file)
+    it attempts to add write permission and then retries.
+    If the error is for another reason it re-raises the error.
+
+    Usage : ``shutil.rmtree(path, onerror=onerror)``.
+
+    :param func: Function which raised the exception.
+    :type func: function
+    :param path: The path name passed to ``shutil.rmtree`` function.
+    :type path: str
+    :param exc_info: Information about the last raised exception.
+    :type exc_info: tuple
+    :rtype: None
+    """
+    import stat
+    if not os.access(path, os.W_OK):
+        os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
+        func(path)
+    else:
+        raise
+
+
 class Error(errors.Error):
     """Class for describing errors during packaging operations.
 
@@ -976,8 +1003,16 @@ class Registry():
 
         # TODO(CD): implement these checks in pure Python?
         scripts_dir = os.path.join(self._base_path, 'scripts')
-        validate_script = os.path.join(scripts_dir, '1-validate-packages.sh')
-        result = subprocess.call(validate_script)
+        if util.is_windows_platform():
+            validate_script = os.path.join(scripts_dir,
+                                           '1-validate-packages.ps1')
+            cmd = ['powershell', '-ExecutionPolicy',
+                   'ByPass', '-File', validate_script]
+            result = subprocess.call(cmd)
+        else:
+            validate_script = os.path.join(scripts_dir,
+                                           '1-validate-packages.sh')
+            result = subprocess.call(validate_script)
         if result is not 0:
             return [Error(
                 'Source tree is not valid [{}]'.format(self._base_path))]

dcos/util.py

@@ -158,7 +158,8 @@ def which(program):
             exe_file = os.path.join(path, program)
             if is_exe(exe_file):
                 return exe_file
-
+    if is_windows_platform() and not program.endswith('.exe'):
+        return which(program + '.exe')
     return None
 
 
@@ -169,8 +170,7 @@ def dcos_path():
     :rtype: str
     """
 
-    dcos_bin_dir = os.path.realpath(sys.argv[0])
-    return os.path.dirname(os.path.dirname(dcos_bin_dir))
+    return os.path.dirname(os.path.dirname(os.sys.executable))
 
 
 def configure_logger_from_environ():

tox.win.ini

@@ -0,0 +1,30 @@
+[tox]
+envlist = py{27,34}-unit
+
+[testenv]
+setenv =
+  DCOS_CONFIG = {env:DCOS_CONFIG}
+  EXHIBITOR_URL = {env:EXHIBITOR_URL}
+deps =
+  teamcity-messages
+  pytest
+  pytest-cov
+  pypiwin32
+
+[testenv:syntax]
+deps =
+  teamcity-messages
+  flake8
+  isort
+
+commands =
+  flake8 --verbose {env:CI_FLAGS:} dcos tests setup.py
+  isort --recursive --check-only --diff --verbose dcos tests setup.py
+
+[testenv:py27-unit]
+commands =
+  py.test -vv {env:CI_FLAGS:} --cov {envsitepackagesdir}/dcos tests
+
+[testenv:py34-unit]
+commands =
+  py.test -vv {env:CI_FLAGS:} --cov {envsitepackagesdir}/dcos tests

win_bin/clean.ps1

@@ -0,0 +1,5 @@
+$BaseDir = (Get-Location).Path
+Remove-Item $BASEDIR\env -Recurse -Force -erroraction 'silentlycontinue'
+Remove-Item $BASEDIR\dist -Recurse -Force -erroraction 'silentlycontinue'
+Remove-Item $BASEDIR\build -Recurse -Force -erroraction 'silentlycontinue'
+Get-ChildItem -Path $BaseDir -Filter *.pyc -Recurse | Remove-Item -Force

win_bin/env.ps1

@@ -0,0 +1,22 @@
+$BaseDir = (Get-Location).Path
+if (-Not (Test-Path $BaseDir\env -PathType Container)) {
+    virtualenv -q $BASEDIR/env --prompt='(dcoscli) '
+    echo "Virtualenv created."
+
+    & $BASEDIR/env/Scripts/activate
+    echo "Virtualenv activated."
+
+    & $BASEDIR/env/Scripts/pip.exe install -r $BASEDIR/requirements.txt
+    & $BASEDIR/env/Scripts/pip.exe install -e $BASEDIR
+    echo "Requirements installed."
+}
+ElseIf ((Test-Path $BASEDIR/env/bin/activate )) {
+
+    & $BASEDIR/env/Scripts/activate
+    echo "Virtualenv activated."
+
+    & $BASEDIR/env/Scripts/pip install -r $BASEDIR/requirements.txt
+    & $BASEDIR/env/Scripts/pip install -e $BASEDIR
+    echo "Requirements installed."
+
+}

win_bin/packages.ps1

@@ -0,0 +1,7 @@
+$BaseDir = (Get-Location).Path
+
+echo "Building wheel..."
+& "$BASEDIR\env\Scripts\python.exe" setup.py bdist_wheel
+
+echo "Building egg..."
+& "$BASEDIR\env\Scripts\python.exe" setup.py sdist

win_bin/start_tests.ps1

@@ -0,0 +1,16 @@
+$BaseDir = (Get-Location).Path
+& $BaseDir\win_bin\clean.ps1
+& $BaseDir\win_bin\env.ps1
+& $BaseDir\win_bin\packages.ps1
+& $BaseDir\env\Scripts\activate
+tox -c tox.win.ini
+$DcosCliExitCode = $LastExitCode
+& $BaseDir\env\Scripts\deactivate
+cd cli
+& $BaseDir\cli\win_bin\clean.ps1
+& $BaseDir\cli\win_bin\env.ps1
+& $BaseDir\cli\env\Scripts\activate
+tox -c tox.win.ini
+$CliExitCode = $LastExitCode
+& $BaseDir\cli\env\Scripts\deactivate
+exit ($DcosCliExitCode -or $CliExitCode)

win_bin/test.ps1

@@ -0,0 +1,7 @@
+$BaseDir = (Get-Location).Path
+
+cd $BASEDIR
+& $BASEDIR\env\Scripts\activate
+echo "Virtualenv activated."
+
+tox