From 937a5d7b32c2fa4651db81680476fa40a98b8354 Mon Sep 17 00:00:00 2001 From: ytalashko Date: Tue, 16 Jun 2015 00:03:33 +0300 Subject: [PATCH] [ADD] DCOS-1002 Windows integration tests --- cli/tests/integrations/common.py | 4 +- cli/tests/integrations/test_analytics.py | 116 +++++++++--------- cli/tests/integrations/test_config.py | 12 +- cli/tests/integrations/test_dcos.py | 11 +- cli/tests/integrations/test_marathon.py | 6 +- cli/tox.win.ini | 41 +++++++ cli/win_bin/clean.ps1 | 5 + cli/win_bin/env.ps1 | 22 ++++ cli/win_bin/packages.ps1 | 7 ++ cli/win_bin/test.ps1 | 8 ++ dcos/package.py | 43 ++++++- dcos/util.py | 6 +- tox.win.ini | 30 +++++ win_bin/clean.ps1 | 5 + win_bin/env.ps1 | 22 ++++ {bin => win_bin}/install/install-dcos-cli.ps1 | 0 win_bin/packages.ps1 | 7 ++ win_bin/start_tests.ps1 | 16 +++ win_bin/test.ps1 | 7 ++ 19 files changed, 292 insertions(+), 76 deletions(-) create mode 100644 cli/tox.win.ini create mode 100644 cli/win_bin/clean.ps1 create mode 100644 cli/win_bin/env.ps1 create mode 100644 cli/win_bin/packages.ps1 create mode 100644 cli/win_bin/test.ps1 create mode 100644 tox.win.ini create mode 100644 win_bin/clean.ps1 create mode 100644 win_bin/env.ps1 rename {bin => win_bin}/install/install-dcos-cli.ps1 (100%) create mode 100644 win_bin/packages.ps1 create mode 100644 win_bin/start_tests.ps1 create mode 100644 win_bin/test.ps1 diff --git a/cli/tests/integrations/common.py b/cli/tests/integrations/common.py index 6d658ac..6478a29 100644 --- a/cli/tests/integrations/common.py +++ b/cli/tests/integrations/common.py @@ -30,7 +30,9 @@ def exec_command(cmd, env=None, stdin=None): stderr=subprocess.PIPE, env=env) - stdout, stderr = process.communicate() + # This is needed to get rid of '\r' from Windows's lines endings. + stdout, stderr = [std_stream.replace(b'\r', b'') + for std_stream in process.communicate()] # We should always print the stdout and stderr print('STDOUT: {}'.format(stdout.decode('utf-8'))) diff --git a/cli/tests/integrations/test_analytics.py b/cli/tests/integrations/test_analytics.py index ac7665d..5e507c2 100644 --- a/cli/tests/integrations/test_analytics.py +++ b/cli/tests/integrations/test_analytics.py @@ -35,6 +35,64 @@ def _mock(fn): return wrapper +@_mock +def test_production_setting_true(): + '''Test that env var DCOS_PRODUCTION as empty string sends exceptions + to the 'prod' environment. + + ''' + + args = [util.which('dcos')] + env = _env_reporting() + env['DCOS_PRODUCTION'] = '' + + with patch('sys.argv', args), patch.dict(os.environ, env): + assert main() == 0 + + _, kwargs = requests.post.call_args_list[0] + assert kwargs['auth'].username == SEGMENT_IO_WRITE_KEY_PROD + + rollbar.init.assert_called_with(ROLLBAR_SERVER_POST_KEY, 'prod') + + +@_mock +def test_production_setting_false(): + '''Test that env var DCOS_PRODUCTION=false sends exceptions to + the 'dev' environment. + + ''' + + args = [util.which('dcos')] + env = _env_reporting() + env['DCOS_PRODUCTION'] = 'false' + + with patch('sys.argv', args), patch.dict(os.environ, env): + assert main() == 0 + + _, kwargs = requests.post.call_args_list[0] + assert kwargs['auth'].username == SEGMENT_IO_WRITE_KEY_DEV + + rollbar.init.assert_called_with(ROLLBAR_SERVER_POST_KEY, 'dev') + + +@_mock +def test_config_set(): + '''Tests that a `dcos config set core.email ` makes a + segment.io identify call''' + + args = [util.which('dcos'), 'config', 'set', 'core.email', 'test@mail.com'] + env = _env_reporting() + + with patch('sys.argv', args), patch.dict(os.environ, env): + assert config_main() == 0 + + # segment.io + assert mock_called_some_args(requests.post, + '{}/identify'.format(SEGMENT_URL), + json={'userId': 'test@mail.com'}, + timeout=1) + + @_mock def test_no_exc(): '''Tests that a command which does not raise an exception does not @@ -115,64 +173,6 @@ def test_config_reporting_false(): assert requests.post.call_count == 0 -@_mock -def test_production_setting_true(): - '''Test that env var DCOS_PRODUCTION as empty string sends exceptions - to the 'prod' environment. - - ''' - - args = [util.which('dcos')] - env = _env_reporting() - env['DCOS_PRODUCTION'] = '' - - with patch('sys.argv', args), patch.dict(os.environ, env): - assert main() == 0 - - _, kwargs = requests.post.call_args_list[0] - assert kwargs['auth'].username == SEGMENT_IO_WRITE_KEY_PROD - - rollbar.init.assert_called_with(ROLLBAR_SERVER_POST_KEY, 'prod') - - -@_mock -def test_production_setting_false(): - '''Test that env var DCOS_PRODUCTION=false sends exceptions to - the 'dev' environment. - - ''' - - args = [util.which('dcos')] - env = _env_reporting() - env['DCOS_PRODUCTION'] = 'false' - - with patch('sys.argv', args), patch.dict(os.environ, env): - assert main() == 0 - - _, kwargs = requests.post.call_args_list[0] - assert kwargs['auth'].username == SEGMENT_IO_WRITE_KEY_DEV - - rollbar.init.assert_called_with(ROLLBAR_SERVER_POST_KEY, 'dev') - - -@_mock -def test_config_set(): - '''Tests that a `dcos config set core.email ` makes a - segment.io identify call''' - - args = [util.which('dcos'), 'config', 'set', 'core.email', 'test@mail.com'] - env = _env_reporting() - - with patch('sys.argv', args), patch.dict(os.environ, env): - assert config_main() == 0 - - # segment.io - assert mock_called_some_args(requests.post, - '{}/identify'.format(SEGMENT_URL), - json={'userId': 'test@mail.com'}, - timeout=1) - - def _env_reporting(): path = os.path.join('tests', 'data', 'analytics', 'dcos_reporting.toml') return {constants.DCOS_CONFIG_ENV: path} diff --git a/cli/tests/integrations/test_config.py b/cli/tests/integrations/test_config.py index 5706f8a..0cdcd6b 100644 --- a/cli/tests/integrations/test_config.py +++ b/cli/tests/integrations/test_config.py @@ -12,20 +12,24 @@ from .common import assert_command, exec_command @pytest.fixture def env(): - return { + r = os.environ.copy() + r.update({ constants.PATH_ENV: os.environ[constants.PATH_ENV], constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "dcos.toml"), cli_constants.DCOS_PRODUCTION_ENV: 'false' - } + }) + return r @pytest.fixture def missing_env(): - return { + r = os.environ.copy() + r.update({ constants.PATH_ENV: os.environ[constants.PATH_ENV], constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "missing_params_dcos.toml") - } + }) + return r def test_help(): diff --git a/cli/tests/integrations/test_dcos.py b/cli/tests/integrations/test_dcos.py index 28384f8..8830781 100644 --- a/cli/tests/integrations/test_dcos.py +++ b/cli/tests/integrations/test_dcos.py @@ -71,9 +71,11 @@ def test_version(): def test_missing_dcos_config(): - env = { + env = os.environ.copy() + del env['DCOS_CONFIG'] + env.update({ constants.PATH_ENV: os.environ[constants.PATH_ENV], - } + }) stdout = (b"Environment variable 'DCOS_CONFIG' must be set " b"to the DCOS config file.\n") @@ -85,10 +87,11 @@ def test_missing_dcos_config(): def test_dcos_config_not_a_file(): - env = { + env = os.environ.copy() + env.update({ constants.PATH_ENV: os.environ[constants.PATH_ENV], 'DCOS_CONFIG': 'missing/file', - } + }) stdout = (b"Environment variable 'DCOS_CONFIG' maps to " b"'missing/file' and it is not a file.\n") diff --git a/cli/tests/integrations/test_marathon.py b/cli/tests/integrations/test_marathon.py index 3225d0d..9cdf0f2 100644 --- a/cli/tests/integrations/test_marathon.py +++ b/cli/tests/integrations/test_marathon.py @@ -134,11 +134,13 @@ def test_about(): @pytest.fixture def missing_env(): - return { + env = os.environ.copy() + env.update({ constants.PATH_ENV: os.environ[constants.PATH_ENV], constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "missing_marathon_params.toml") - } + }) + return env def test_missing_config(missing_env): diff --git a/cli/tox.win.ini b/cli/tox.win.ini new file mode 100644 index 0000000..ea9d152 --- /dev/null +++ b/cli/tox.win.ini @@ -0,0 +1,41 @@ +[tox] +envlist = py{27,34}-unit, py{27,34}-integration + +[testenv] +setenv = + DCOS_CONFIG = {env:DCOS_CONFIG} + EXHIBITOR_URL = {env:EXHIBITOR_URL} +deps = + teamcity-messages + pytest + pytest-cov + mock + pypiwin32 + -e.. + +[testenv:syntax] +deps = + teamcity-messages + flake8 + isort + .. + +commands = + flake8 --verbose {env:CI_FLAGS:} dcoscli tests setup.py + isort --recursive --check-only --diff --verbose dcoscli tests setup.py + +[testenv:py27-integration] +commands = + py.test -vv {env:CI_FLAGS:} tests/integrations{posargs} + +[testenv:py34-integration] +commands = + py.test -vv {env:CI_FLAGS:} tests/integrations{posargs} + +[testenv:py27-unit] +commands = + py.test -vv {env:CI_FLAGS:} tests/unit{posargs} + +[testenv:py34-unit] +commands = + py.test -vv {env:CI_FLAGS:} tests/unit{posargs} diff --git a/cli/win_bin/clean.ps1 b/cli/win_bin/clean.ps1 new file mode 100644 index 0000000..dd00602 --- /dev/null +++ b/cli/win_bin/clean.ps1 @@ -0,0 +1,5 @@ +$BaseDir = (Get-Location).Path +Remove-Item $BASEDIR\env -Recurse -Force -erroraction 'silentlycontinue' +Remove-Item $BASEDIR\dist -Recurse -Force -erroraction 'silentlycontinue' +Remove-Item $BASEDIR\build -Recurse -Force -erroraction 'silentlycontinue' +Get-ChildItem -Path $BaseDir -Filter *.pyc -Recurse | Remove-Item -Force \ No newline at end of file diff --git a/cli/win_bin/env.ps1 b/cli/win_bin/env.ps1 new file mode 100644 index 0000000..dcda30e --- /dev/null +++ b/cli/win_bin/env.ps1 @@ -0,0 +1,22 @@ +$BaseDir = (Get-Location).Path +if (-Not (Test-Path $BaseDir\env -PathType Container)) { + virtualenv -q $BASEDIR/env --prompt='(dcoscli) ' + echo "Virtualenv created." + + & $BASEDIR/env/Scripts/activate + echo "Virtualenv activated." + + & $BASEDIR/env/Scripts/pip.exe install -r $BASEDIR/requirements.txt + & $BASEDIR/env/Scripts/pip.exe install -e $BASEDIR + echo "Requirements installed." +} +ElseIf ((Test-Path $BASEDIR/env/bin/activate )) { + + & $BASEDIR/env/Scripts/activate + echo "Virtualenv activated." + + & $BASEDIR/env/Scripts/pip install -r $BASEDIR/requirements.txt + & $BASEDIR/env/Scripts/pip install -e $BASEDIR + echo "Requirements installed." + +} diff --git a/cli/win_bin/packages.ps1 b/cli/win_bin/packages.ps1 new file mode 100644 index 0000000..1ff9843 --- /dev/null +++ b/cli/win_bin/packages.ps1 @@ -0,0 +1,7 @@ +$BaseDir = (Get-Location).Path + +echo "Building wheel..." +"$BASEDIR/env/bin/python" setup.py bdist_wheel + +echo "Building egg..." +"$BASEDIR/env/bin/python" setup.py sdist diff --git a/cli/win_bin/test.ps1 b/cli/win_bin/test.ps1 new file mode 100644 index 0000000..b55dec1 --- /dev/null +++ b/cli/win_bin/test.ps1 @@ -0,0 +1,8 @@ +$BaseDir = (Get-Location).Path + +cd $BASEDIR +& $BASEDIR\env\Scripts\activate +echo "Virtualenv activated." + +tox -c tox.win.ini +exit $LastExitCode diff --git a/dcos/package.py b/dcos/package.py index c9f838d..51b7296 100644 --- a/dcos/package.py +++ b/dcos/package.py @@ -714,7 +714,9 @@ def update_sources(config, validate=False): target_dir = os.path.join(cache_dir, source.hash()) try: if os.path.exists(target_dir): - shutil.rmtree(target_dir, ignore_errors=False) + shutil.rmtree(target_dir, + onerror=_rmtree_on_error, + ignore_errors=False) except OSError: err = Error( 'Could not remove directory [{}]'.format(target_dir)) @@ -935,7 +937,8 @@ PATH = {}""".format(os.environ[constants.PATH_ENV])) branch='master') # Remove .git directory to save space. - shutil.rmtree(os.path.join(target_dir, ".git")) + shutil.rmtree(os.path.join(target_dir, ".git"), + onerror=_rmtree_on_error) return None except git.exc.GitCommandError: @@ -943,6 +946,30 @@ PATH = {}""".format(os.environ[constants.PATH_ENV])) 'Unable to fetch packages from [{}]'.format(self.url)) +def _rmtree_on_error(func, path, exc_info): + """Error handler for ``shutil.rmtree``. + If the error is due to an access error (read only file) + it attempts to add write permission and then retries. + If the error is for another reason it re-raises the error. + + Usage : ``shutil.rmtree(path, onerror=onerror)``. + + :param func: Function which raised the exception. + :type func: function + :param path: The path name passed to ``shutil.rmtree`` function. + :type path: str + :param exc_info: Information about the last raised exception. + :type exc_info: tuple + :rtype: None + """ + import stat + if not os.access(path, os.W_OK): + os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + func(path) + else: + raise + + class Error(errors.Error): """Class for describing errors during packaging operations. @@ -985,8 +1012,16 @@ class Registry(): # TODO(CD): implement these checks in pure Python? scripts_dir = os.path.join(self._base_path, 'scripts') - validate_script = os.path.join(scripts_dir, '1-validate-packages.sh') - result = subprocess.call(validate_script) + if util.is_windows_platform(): + validate_script = os.path.join(scripts_dir, + '1-validate-packages.ps1') + cmd = ['powershell', '-ExecutionPolicy', + 'ByPass', '-File', validate_script] + result = subprocess.call(cmd) + else: + validate_script = os.path.join(scripts_dir, + '1-validate-packages.sh') + result = subprocess.call(validate_script) if result is not 0: return [Error( 'Source tree is not valid [{}]'.format(self._base_path))] diff --git a/dcos/util.py b/dcos/util.py index 80e3cd5..c2359d9 100644 --- a/dcos/util.py +++ b/dcos/util.py @@ -158,7 +158,8 @@ def which(program): exe_file = os.path.join(path, program) if is_exe(exe_file): return exe_file - + if is_windows_platform() and not program.endswith('.exe'): + return which(program + '.exe') return None @@ -169,8 +170,7 @@ def dcos_path(): :rtype: str """ - dcos_bin_dir = os.path.realpath(sys.argv[0]) - return os.path.dirname(os.path.dirname(dcos_bin_dir)) + return os.path.dirname(os.path.dirname(os.sys.executable)) def configure_logger_from_environ(): diff --git a/tox.win.ini b/tox.win.ini new file mode 100644 index 0000000..842e1ea --- /dev/null +++ b/tox.win.ini @@ -0,0 +1,30 @@ +[tox] +envlist = py{27,34}-unit + +[testenv] +setenv = + DCOS_CONFIG = {env:DCOS_CONFIG} + EXHIBITOR_URL = {env:EXHIBITOR_URL} +deps = + teamcity-messages + pytest + pytest-cov + pypiwin32 + +[testenv:syntax] +deps = + teamcity-messages + flake8 + isort + +commands = + flake8 --verbose {env:CI_FLAGS:} dcos tests setup.py + isort --recursive --check-only --diff --verbose dcos tests setup.py + +[testenv:py27-unit] +commands = + py.test -vv {env:CI_FLAGS:} --cov {envsitepackagesdir}/dcos tests + +[testenv:py34-unit] +commands = + py.test -vv {env:CI_FLAGS:} --cov {envsitepackagesdir}/dcos tests diff --git a/win_bin/clean.ps1 b/win_bin/clean.ps1 new file mode 100644 index 0000000..dd00602 --- /dev/null +++ b/win_bin/clean.ps1 @@ -0,0 +1,5 @@ +$BaseDir = (Get-Location).Path +Remove-Item $BASEDIR\env -Recurse -Force -erroraction 'silentlycontinue' +Remove-Item $BASEDIR\dist -Recurse -Force -erroraction 'silentlycontinue' +Remove-Item $BASEDIR\build -Recurse -Force -erroraction 'silentlycontinue' +Get-ChildItem -Path $BaseDir -Filter *.pyc -Recurse | Remove-Item -Force \ No newline at end of file diff --git a/win_bin/env.ps1 b/win_bin/env.ps1 new file mode 100644 index 0000000..9980953 --- /dev/null +++ b/win_bin/env.ps1 @@ -0,0 +1,22 @@ +$BaseDir = (Get-Location).Path +if (-Not (Test-Path $BaseDir\env -PathType Container)) { + virtualenv -q $BASEDIR/env --prompt='(dcoscli) ' + echo "Virtualenv created." + + & $BASEDIR/env/Scripts/activate + echo "Virtualenv activated." + + & $BASEDIR/env/Scripts/pip.exe install -r $BASEDIR/requirements.txt + & $BASEDIR/env/Scripts/pip.exe install -e $BASEDIR + echo "Requirements installed." +} +ElseIf ((Test-Path $BASEDIR/env/bin/activate )) { + + & $BASEDIR/env/Scripts/activate + echo "Virtualenv activated." + + & $BASEDIR/env/Scripts/pip install -r $BASEDIR/requirements.txt + & $BASEDIR/env/Scripts/pip install -e $BASEDIR + echo "Requirements installed." + +} \ No newline at end of file diff --git a/bin/install/install-dcos-cli.ps1 b/win_bin/install/install-dcos-cli.ps1 similarity index 100% rename from bin/install/install-dcos-cli.ps1 rename to win_bin/install/install-dcos-cli.ps1 diff --git a/win_bin/packages.ps1 b/win_bin/packages.ps1 new file mode 100644 index 0000000..f1ce450 --- /dev/null +++ b/win_bin/packages.ps1 @@ -0,0 +1,7 @@ +$BaseDir = (Get-Location).Path + +echo "Building wheel..." +& "$BASEDIR\env\Scripts\python.exe" setup.py bdist_wheel + +echo "Building egg..." +& "$BASEDIR\env\Scripts\python.exe" setup.py sdist diff --git a/win_bin/start_tests.ps1 b/win_bin/start_tests.ps1 new file mode 100644 index 0000000..341bfc1 --- /dev/null +++ b/win_bin/start_tests.ps1 @@ -0,0 +1,16 @@ +$BaseDir = (Get-Location).Path +& $BaseDir\win_bin\clean.ps1 +& $BaseDir\win_bin\env.ps1 +& $BaseDir\win_bin\packages.ps1 +& $BaseDir\env\Scripts\activate +tox -c tox.win.ini +$DcosCliExitCode = $LastExitCode +& $BaseDir\env\Scripts\deactivate +cd cli +& $BaseDir\cli\win_bin\clean.ps1 +& $BaseDir\cli\win_bin\env.ps1 +& $BaseDir\cli\env\Scripts\activate +tox -c tox.win.ini +$CliExitCode = $LastExitCode +& $BaseDir\cli\env\Scripts\deactivate +exit ($DcosCliExitCode -or $CliExitCode) diff --git a/win_bin/test.ps1 b/win_bin/test.ps1 new file mode 100644 index 0000000..a613b56 --- /dev/null +++ b/win_bin/test.ps1 @@ -0,0 +1,7 @@ +$BaseDir = (Get-Location).Path + +cd $BASEDIR +& $BASEDIR\env\Scripts\activate +echo "Virtualenv activated." + +tox \ No newline at end of file