get_build_artifact_as_bytes to support non-json files

new function get_build_artifact_as_bytes to replace get_build_artifact (incompatible with artifacts not json-formatted)
+ use a stream to download binary artifacts

all artifacts are returned as bytes to avoid encoding issues
and an exception NotFoundException is raised in case of missing artifact

Closes-Bug: #1973243
Change-Id: I24ce4ecd854f8a19ed4d760404adb7d1ac6b5509
This commit is contained in:
Guillaume DeMengin 2022-11-26 19:35:37 +01:00 committed by gdemengin
parent a8f0d8cfda
commit 5c44d7c9ec
2 changed files with 142 additions and 5 deletions

View File

@ -544,13 +544,13 @@ class Jenkins(object):
# when accessing .text property # when accessing .text property
return response return response
def _request(self, req): def _request(self, req, stream=None):
r = self._session.prepare_request(req) r = self._session.prepare_request(req)
# requests.Session.send() does not honor env settings by design # requests.Session.send() does not honor env settings by design
# see https://github.com/requests/requests/issues/2807 # see https://github.com/requests/requests/issues/2807
_settings = self._session.merge_environment_settings( _settings = self._session.merge_environment_settings(
r.url, {}, None, self._session.verify, None) r.url, {}, stream, self._session.verify, None)
_settings['timeout'] = self.timeout _settings['timeout'] = self.timeout
return self._session.send(r, **_settings) return self._session.send(r, **_settings)
@ -561,7 +561,14 @@ class Jenkins(object):
''' '''
return self.jenkins_request(req, add_crumb, resolve_auth).text return self.jenkins_request(req, add_crumb, resolve_auth).text
def jenkins_request(self, req, add_crumb=True, resolve_auth=True): def jenkins_open_stream(self, req, add_crumb=True, resolve_auth=True):
'''Return the HTTP response body from a ``requests.Request``.
:returns: ``stream``
'''
return self.jenkins_request(req, add_crumb, resolve_auth, True)
def jenkins_request(self, req, add_crumb=True, resolve_auth=True, stream=None):
'''Utility routine for opening an HTTP request to a Jenkins server. '''Utility routine for opening an HTTP request to a Jenkins server.
:param req: A ``requests.Request`` to submit. :param req: A ``requests.Request`` to submit.
@ -569,6 +576,7 @@ class Jenkins(object):
before submitting. Defaults to ``True``. before submitting. Defaults to ``True``.
:param resolve_auth: If True, maybe add authentication. Defaults to :param resolve_auth: If True, maybe add authentication. Defaults to
``True``. ``True``.
:param stream: If True, return a stream
:returns: A ``requests.Response`` object. :returns: A ``requests.Response`` object.
''' '''
try: try:
@ -578,7 +586,7 @@ class Jenkins(object):
self.maybe_add_crumb(req) self.maybe_add_crumb(req)
return self._response_handler( return self._response_handler(
self._request(req)) self._request(req, stream))
except req_exc.HTTPError as e: except req_exc.HTTPError as e:
# Jenkins's funky authentication means its nigh impossible to # Jenkins's funky authentication means its nigh impossible to
@ -742,6 +750,27 @@ class Jenkins(object):
# This can happen if the artifact is not found # This can happen if the artifact is not found
return None return None
def get_build_artifact_as_bytes(self, name, number, artifact):
"""Get artifacts from job
:param name: Job name, ``str``
:param number: Build number, ``str`` (also accepts ``int``)
:param artifact: Artifact relative path, ``str``
:returns: artifact to download, ``bytes``
"""
folder_url, short_name = self._get_job_folder(name)
try:
with self.jenkins_open_stream(requests.Request(
'GET', self._build_url(BUILD_ARTIFACT, locals()))) as response:
if response.encoding is None:
return response.raw.read()
else:
return response.text.encode(response.encoding)
raise JenkinsException('job[%s] number[%s] does not exist' % (name, number))
except requests.exceptions.HTTPError:
raise JenkinsException('job[%s] number[%s] does not exist' % (name, number))
def get_build_stages(self, name, number): def get_build_stages(self, name, number):
"""Get stages info from job """Get stages info from job

View File

@ -1,7 +1,7 @@
import json import json
import collections import collections
from mock import patch from mock import patch, Mock
import jenkins import jenkins
from tests.base import JenkinsTestBase from tests.base import JenkinsTestBase
@ -812,6 +812,114 @@ class JenkinsBuildArtifactUrlTest(JenkinsTestBase):
'Error in request. Possibly authentication failed [401]: Not Authorised') 'Error in request. Possibly authentication failed [401]: Not Authorised')
class JenkinsBuildArtifactAsBytesUrlTest(JenkinsTestBase):
def streamMock(self, encoding=None, text=None, binary=None):
streamMock = Mock()
streamMock.__exit__ = Mock()
streamMock.__enter__ = Mock()
if encoding is None and text is None and binary is None:
streamMock.__enter__.return_value = None
return streamMock
streamMock.__enter__.return_value = Mock()
streamMock.__enter__.return_value.encoding = encoding
streamMock.__enter__.return_value.text = text
streamMock.__enter__.return_value.raw = Mock()
streamMock.__enter__.return_value.raw.read = Mock()
streamMock.__enter__.return_value.raw.read.return_value = binary
return streamMock
@patch.object(jenkins.Jenkins, 'jenkins_open_stream')
def test_simple_ascii_artifact(self, jenkins_mock):
jenkins_mock.return_value = self.streamMock('utf-8', 'ascii')
ret = self.j.get_build_artifact_as_bytes(u'Test Job', number='52', artifact="filename")
self.assertEqual(ret, b'ascii')
self.assertEqual(
jenkins_mock.call_args[0][0].url,
self.make_url('job/Test%20Job/52/artifact/filename'))
self._check_requests(jenkins_mock.call_args_list)
@patch.object(jenkins.Jenkins, 'jenkins_open_stream')
def test_simple_binary_artifact(self, jenkins_mock):
jenkins_mock.return_value = self.streamMock(binary=b'\0\1\2')
ret = self.j.get_build_artifact_as_bytes(u'Test Job', number='52', artifact="filename")
self.assertEqual(ret, b'\0\1\2')
self.assertEqual(
jenkins_mock.call_args[0][0].url,
self.make_url('job/Test%20Job/52/artifact/filename'))
self._check_requests(jenkins_mock.call_args_list)
@patch.object(jenkins.Jenkins, 'jenkins_open_stream')
def test_in_folder(self, jenkins_mock):
jenkins_mock.return_value = self.streamMock('utf-8', 'ascii')
ret = self.j.get_build_artifact_as_bytes(u'a Folder/Test Job', number='52', artifact="file name")
self.assertEqual(ret, b'ascii')
self.assertEqual(
jenkins_mock.call_args[0][0].url,
self.make_url('job/a%20Folder/job/Test%20Job/52/artifact/file%20name'))
self._check_requests(jenkins_mock.call_args_list)
@patch.object(jenkins.Jenkins, 'jenkins_open_stream')
def test_matrix(self, jenkins_mock):
jenkins_mock.return_value = self.streamMock('utf-8', 'ascii')
ret = self.j.get_build_artifact_as_bytes(u'a Folder/Test Job', number='52/index=matrix',
artifact="file name")
self.assertEqual(ret, b'ascii')
self.assertEqual(
jenkins_mock.call_args[0][0].url,
self.make_url('job/a%20Folder/job/Test%20Job/52/index=matrix/artifact/file%20name'))
self._check_requests(jenkins_mock.call_args_list)
@patch('jenkins.requests.Session.send', autospec=True)
def test_404_item_not_found(self, session_send_mock):
session_send_mock.side_effect = iter([
build_response_mock(404, reason="Not Found"), # crumb
build_response_mock(404, reason="Not Found"), # request
])
with self.assertRaises(jenkins.JenkinsException) as context_manager:
self.j.get_build_artifact_as_bytes(u'TestJob', number='52', artifact="filename")
self.assertEqual(
str(context_manager.exception),
'Requested item could not be found')
@patch.object(jenkins.Jenkins, 'jenkins_open_stream')
def test_open_return_none(self, jenkins_mock):
jenkins_mock.return_value = self.streamMock()
with self.assertRaises(jenkins.JenkinsException) as context_manager:
self.j.get_build_artifact_as_bytes(u'TestJob', number='52', artifact="filename")
self.assertEqual(
str(context_manager.exception),
'job[TestJob] number[52] does not exist')
self._check_requests(jenkins_mock.call_args_list)
@patch('jenkins.requests.Session.send', autospec=True)
def test_raise_HTTPError(self, session_send_mock):
session_send_mock.side_effect = iter([
build_response_mock(401, reason="Not Authorised"), # crumb
build_response_mock(401, reason="Not Authorised"), # request
])
with self.assertRaises(jenkins.JenkinsException) as context_manager:
self.j.get_build_artifact_as_bytes(u'TestJob', number='52', artifact="filename")
self.assertEqual(
str(context_manager.exception),
'Error in request. Possibly authentication failed [401]: Not Authorised')
@patch('jenkins.requests.Session.send', autospec=True)
def test_in_folder_raise_HTTPError(self, session_send_mock):
session_send_mock.side_effect = iter([
build_response_mock(401, reason="Not Authorised"), # crumb
build_response_mock(401, reason="Not Authorised"), # request
])
with self.assertRaises(jenkins.JenkinsException) as context_manager:
self.j.get_build_artifact_as_bytes(u'a Folder/TestJob', number='52', artifact="filename")
self.assertEqual(
str(context_manager.exception),
'Error in request. Possibly authentication failed [401]: Not Authorised')
class JenkinsBuildStagesUrlTest(JenkinsTestBase): class JenkinsBuildStagesUrlTest(JenkinsTestBase):
@patch.object(jenkins.Jenkins, 'jenkins_open') @patch.object(jenkins.Jenkins, 'jenkins_open')