zuul/zuul-jobs
Code Issues Proposed changes

Update upload-logs roles to support endpoint override

Browse Source 
Add zuul_log_storage_proxy_address variable attempting to replace
storage endpoint address with.
The usecase is when the storage proxy is positioned infront of storage
endpoint.

Change-Id: I353cd50b76683212b0319a1e28f34901267c08e4
This commit is contained in:
okozachenko 2021-02-09 15:49:53 +02:00 committed by James E. Blair
parent b1521dbdb5
commit 862ae3f5d6
9 changed files with 87 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
roles/upload-logs-base/library/zuul_google_storage_upload.py
View File

@ -93,7 +93,8 @@ class Uploader():
self.dry_run = dry_run
if dry_run:
self.url = 'http://dry-run-url.com/a/path/'
self.endpoint = 'http://dry-run-url.com'
self.path = '/a/path/'
return
self.client = client
@ -107,9 +108,8 @@ class Uploader():
self.bucket.cors = cors
self.bucket.website = {"mainPageSuffix": "index.html"}
self.bucket.update()
self.url = os.path.join('https://storage.googleapis.com/',
container, self.prefix)
self.endpoint = 'https://storage.googleapis.com'
self.path = os.path.join(container, self.prefix)
def upload(self, file_list):
"""Spin up thread pool to upload to storage"""
@ -240,7 +240,7 @@ def run(container, files,
# Upload.
uploader = Uploader(client, container, prefix, dry_run)
uploader.upload(file_list)
return uploader.url
return uploader.endpoint, uploader.path
def ansible_main():
@ -260,17 +260,18 @@ def ansible_main():
)
p = module.params
url = run(p.get('container'), p.get('files'),
indexes=p.get('indexes'),
parent_links=p.get('parent_links'),
topdir_parent_link=p.get('topdir_parent_link'),
partition=p.get('partition'),
footer=p.get('footer'),
prefix=p.get('prefix'),
credentials_file=p.get('credentials_file'),
project=p.get('project'))
endpoint, path = run(p.get('container'), p.get('files'),
indexes=p.get('indexes'),
parent_links=p.get('parent_links'),
topdir_parent_link=p.get('topdir_parent_link'),
partition=p.get('partition'),
footer=p.get('footer'),
prefix=p.get('prefix'),
credentials_file=p.get('credentials_file'),
project=p.get('project'))
module.exit_json(changed=True,
url=url)
endpoint=endpoint,
path=path)
def cli_main():
@ -320,17 +321,17 @@ def cli_main():
if append_footer.lower() == 'none':
append_footer = None
url = run(args.container, args.files,
indexes=not args.no_indexes,
parent_links=not args.no_parent_links,
topdir_parent_link=args.create_topdir_parent_link,
partition=args.partition,
footer=append_footer,
prefix=args.prefix,
dry_run=args.dry_run,
credentials_file=args.credentials_file,
project=args.project)
print(url)
_, path = run(args.container, args.files,
indexes=not args.no_indexes,
parent_links=not args.no_parent_links,
topdir_parent_link=args.create_topdir_parent_link,
partition=args.partition,
footer=append_footer,
prefix=args.prefix,
dry_run=args.dry_run,
credentials_file=args.credentials_file,
project=args.project)
print(path)
if __name__ == '__main__':

49
roles/upload-logs-base/library/zuul_s3_upload.py
View File

@ -63,7 +63,8 @@ class Uploader():
self.dry_run = dry_run
self.public = public
if dry_run:
self.url = 'http://dry-run-url.com/a/path/'
self.endpoint = 'http://dry-run-url.com'
self.path = '/a/path'
return
self.prefix = prefix or ''
@ -73,8 +74,7 @@ class Uploader():
else:
self.endpoint = 'https://s3.amazonaws.com/'
self.url = os.path.join(self.endpoint,
bucket, self.prefix)
self.path = os.path.join(bucket, self.prefix)
self.s3 = boto3.resource('s3',
endpoint_url=self.endpoint,
@ -223,7 +223,7 @@ def run(bucket, public, files, endpoint=None,
aws_secret_key=aws_secret_key)
upload_failures = uploader.upload(file_list)
return uploader.url, upload_failures
return uploader.endpoint, uploader.path, upload_failures
def ansible_main():
@ -245,24 +245,28 @@ def ansible_main():
)
p = module.params
url, failures = run(p.get('bucket'),
p.get('public'),
p.get('files'),
p.get('endpoint'),
indexes=p.get('indexes'),
parent_links=p.get('parent_links'),
topdir_parent_link=p.get('topdir_parent_link'),
partition=p.get('partition'),
footer=p.get('footer'),
prefix=p.get('prefix'),
aws_access_key=p.get('aws_access_key'),
aws_secret_key=p.get('aws_secret_key'))
endpoint, path, failures = run(
p.get('bucket'),
p.get('public'),
p.get('files'),
p.get('endpoint'),
indexes=p.get('indexes'),
parent_links=p.get('parent_links'),
topdir_parent_link=p.get('topdir_parent_link'),
partition=p.get('partition'),
footer=p.get('footer'),
prefix=p.get('prefix'),
aws_access_key=p.get('aws_access_key'),
aws_secret_key=p.get('aws_secret_key'),
)
if failures:
module.fail_json(changed=True,
url=url,
endpoint=endpoint,
path=path,
failures=failures)
module.exit_json(changed=True,
url=url,
endpoint=endpoint,
path=path,
failures=failures)
@ -290,11 +294,10 @@ def cli_main():
logging.basicConfig(level=logging.DEBUG)
logging.captureWarnings(True)
url = run(args.bucket, args.files,
prefix=args.prefix,
public=not args.no_public,
endpoint=args.endpoint)
print(url)
_, path, _ = run(args.bucket, not args.no_public, args.files,
prefix=args.prefix,
endpoint=args.endpoint)
print(path)
if __name__ == '__main__':

18
roles/upload-logs-base/library/zuul_swift_upload.py
View File

@ -77,7 +77,8 @@ class Uploader():
self.dry_run = dry_run
if dry_run:
self.url = 'http://dry-run-url.com/a/path/'
self.endpoint = 'http://dry-run-url.com'
self.path = '/a/path'
return
self.cloud = cloud
@ -140,8 +141,8 @@ class Uploader():
else:
endpoint = self.cloud.object_store.get_endpoint()
container = os.path.join(endpoint, self.container)
self.url = os.path.join(container, self.prefix)
self.endpoint = endpoint
self.path = os.path.join(self.container, self.prefix)
def upload(self, file_list):
"""Spin up thread pool to upload to swift"""
@ -288,7 +289,7 @@ def run(cloud, container, files,
uploader = Uploader(cloud, container, prefix, delete_after,
public, dry_run)
upload_failures = uploader.upload(file_list)
return uploader.url, upload_failures
return uploader.endpoint, uploader.path, upload_failures
def ansible_main():
@ -311,7 +312,7 @@ def ansible_main():
p = module.params
cloud = get_cloud(p.get('cloud'))
try:
url, upload_failures = run(
endpoint, path, upload_failures = run(
cloud, p.get('container'), p.get('files'),
indexes=p.get('indexes'),
parent_links=p.get('parent_links'),
@ -334,7 +335,8 @@ def ansible_main():
region_name=cloud.config.region_name)
module.exit_json(
changed=True,
url=url,
endpoint=endpoint,
path=path,
upload_failures=upload_failures,
)
@ -394,7 +396,7 @@ def cli_main():
if append_footer.lower() == 'none':
append_footer = None
url, _ = run(
_, path, _ = run(
get_cloud(args.cloud), args.container, args.files,
indexes=not args.no_indexes,
parent_links=not args.no_parent_links,
@ -406,7 +408,7 @@ def cli_main():
public=not args.no_public,
dry_run=args.dry_run
)
print(url)
print(path)
if __name__ == '__main__':

7
roles/upload-logs-gcs/README.rst
View File

@ -78,3 +78,10 @@ Google Cloud Application Default Credentials.
:zuul:rolevar:`upload-logs-gcs.zuul_log_credentials_file`, the name
of the Google Cloud project of the log container must also be
supplied.
.. zuul:rolevar:: zuul_log_storage_proxy_url
The url of the proxy for the cloud object store.
If you are using zuul-storage-proxy to proxy requests for logs, set this
to the the URL of the log proxy server. When set, this role will replace
the cloud storage endpoint with this value in the log URL returned to Zuul.

2
roles/upload-logs-gcs/tasks/main.yaml
View File

@ -34,5 +34,5 @@
zuul_return:
data:
zuul:
log_url: "{{ upload_results.url }}/"
log_url: "{{ zuul_log_storage_proxy_url | default(upload_results.endpoint, true) | regex_replace('\\/$', '') }}/{{ upload_results.path }}"
when: upload_results is defined

7
roles/upload-logs-s3/README.rst
View File

@ -76,3 +76,10 @@ installed in the Ansible environment on the Zuul executor.
The endpoint to use when uploading logs to an s3 compatible service.
By default this will be automatically constructed by boto but should be set when working with non-aws hosted s3 service.
.. zuul:rolevar:: zuul_log_storage_proxy_url
The url of the proxy for the cloud object store.
If you are using zuul-storage-proxy to proxy requests for logs, set this
to the the URL of the log proxy server. When set, this role will replace
the cloud storage endpoint with this value in the log URL returned to Zuul.

2
roles/upload-logs-s3/tasks/main.yaml
View File

@ -36,5 +36,5 @@
zuul_return:
data:
zuul:
log_url: "{{ upload_results.url }}/"
log_url: "{{ zuul_log_storage_proxy_url | default(upload_results.endpoint, true) | regex_replace('\\/$', '') }}/{{ upload_results.path }}"
when: upload_results is defined

7
roles/upload-logs-swift/README.rst
View File

@ -76,3 +76,10 @@ This uploads logs to an OpenStack Object Store (Swift) container.
More details can be found at
:zuul:rolevar:`set-zuul-log-path-fact.zuul_log_path_shard_build`.
.. zuul:rolevar:: zuul_log_storage_proxy_url
The url of the proxy for the cloud object store.
If you are using zuul-storage-proxy to proxy requests for logs, set this
to the the URL of the log proxy server. When set, this role will replace
the cloud storage endpoint with this value in the log URL returned to Zuul.

2
roles/upload-logs-swift/tasks/main.yaml
View File

@ -36,7 +36,7 @@
zuul_return:
data:
zuul:
log_url: "{{ upload_results.url }}/"
log_url: "{{ zuul_log_storage_proxy_url | default(upload_results.endpoint, true) | regex_replace('\\/$', '') }}/{{ upload_results.path }}"
- name: Print upload failures
debug:
var: upload_results.upload_failures