Create region retries when Ranger server not available

Change-Id: Ice06d39e465fa70593920d5bf7b18d1a427dadbf
This commit is contained in:
Chi Lo 2020-09-11 07:40:55 -07:00
parent b5cbc37078
commit 743f15efbc
3 changed files with 28 additions and 8 deletions

View File

@ -104,10 +104,7 @@ def build_payload():
{"publicURL": os.environ['ORD_ENDPOINT'],
"type": "ord"}]
payload = json.dumps(payload)
sys.stdout.write("Json dumps: %s\n" % payload)
return payload
return json.dumps(payload)
except Exception as exp:
sys.stderr.write("Exp: Error building payload: %s\n" % str(exp))
@ -122,19 +119,22 @@ def notify_ranger_create_region(payload):
url = os.environ['RMS_ENDPOINT']
# Retry up to 5 times
for i in range(5):
time.sleep(15)
done = False
# Retry up to 3 times
for i in range(3):
time.sleep(15) if i != 0 else None
try:
resp = requests.post(
url, data=payload, headers=headers, timeout=120)
if resp.status_code == 409:
sys.stdout.write("Region already existed\n")
done = True
break
elif resp.status_code == 201:
result = resp.json()
sys.stdout.write("Region created successfully: %s\n" % result)
done = True
break
else:
sys.stderr.write("Failed to create region: %s - %s\n" %
@ -151,7 +151,21 @@ def notify_ranger_create_region(payload):
sys.stderr.write("Unknown Exp: %s\n" % str(ex))
continue
return done
if __name__ == "__main__":
payload = build_payload()
notify_ranger_create_region(payload)
sys.stdout.write("Json dumps: %s\n" % payload)
sys.stdout.flush()
duration = int(os.environ['RETRY_DURATION_MINUTES'])
interval = int(os.environ['RETRY_INTERVAL_MINUTES'])
retries = duration//interval
for retry in range(retries):
if notify_ranger_create_region(payload):
break
else:
sys.stderr.flush()
time.sleep(interval*60)

View File

@ -74,6 +74,10 @@ spec:
value: {{ tuple "ranger_agent" "public" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | trimSuffix .Values.endpoints.ranger_agent.path.default }}
- name: RMS_ENDPOINT
value: {{ .Values.conf.region.rms_listener_endpoint }}
- name: RETRY_DURATION_MINUTES
value: {{ .Values.conf.region.retry_duration_minutes | quote }}
- name: RETRY_INTERVAL_MINUTES
value: {{ .Values.conf.region.retry_interval_minutes | quote }}
- name: CAFILE
value: {{ default "" .Values.conf.ranger_agent.keystone_authtoken.https_cacert }}
{{- with $env := dict "ksUserSecret" .Values.secrets.identity.ranger_agent }}

View File

@ -622,6 +622,8 @@ conf:
"OSVersion": "Stein"
}
rms_listener_endpoint: http://localhost:7003/v2/orm/regions
retry_duration_minutes: 660
retry_interval_minutes: 15
rabbitmq:
policies: