Multiple worksheets for Rally json Google Sheets
This patch introduces multiple worksheets within one Google sheet for multiple atomic actions Change-Id: Ia6542cd7646a87c53cd633bde070939729a7ef9b
This commit is contained in:
Sanjay Chari
parent
675396a930
commit
a0d555812b
@ -343,7 +343,7 @@ Generate CSV file/Google Sheets from Rally json file
|
||||
--------------------------------------------
|
||||
Rally generates a json file with data about atomic actions duration from each iteration. These atomic actions often occur multiple times within one iteration.
|
||||
Browbeat has a script which allows a user to generate a CSV file and also has an option to generate a Google Sheet about individual resource
|
||||
duration through the Rally json file. To use the script to upload the CSV file to Google Sheets, a Google Drive service account is required.
|
||||
duration through the Rally json file. To use the script to upload to Google Sheets, a Google Drive service account is required.
|
||||
The script sends an email to the email id of the user with the Google sheet if the --uploadgooglesheet option is enabled.
|
||||
|
||||
To generate only a CSV file and not upload to Google Sheets :
|
||||
@ -351,11 +351,27 @@ To generate only a CSV file and not upload to Google Sheets :
|
||||
::
|
||||
|
||||
$ source .browbeat-venv/bin/activate && cd utils
|
||||
$ python rally_google_sheet_gen.py -c <path to csv file to write locally> -j <path to rally json file> -a <atomic action(Eg.: nova.boot_server>
|
||||
$ python rally_google_sheet_gen.py -c -f <path to directory to write csv file locally>
|
||||
-j <path to rally json file>
|
||||
-a <space separated list of atomic actions(Eg.: boot_server create_network)>
|
||||
|
||||
To only upload to Google Sheets and not generate CSV files :
|
||||
|
||||
::
|
||||
$ source .browbeat-venv/bin/activate && cd utils
|
||||
$ python rally_google_sheet_gen.py
|
||||
-j <path to rally json file>
|
||||
-a <space separated list of atomic actions(Eg.: boot_server create_network)> -g
|
||||
-s <path to google service account json credentials file>
|
||||
-e <email id of user> -n <name of google sheet to be created>
|
||||
|
||||
|
||||
To generate a CSV file and upload to Google Sheets :
|
||||
|
||||
::
|
||||
$ source .browbeat-venv/bin/activate && cd utils
|
||||
$ python rally_google_sheet_gen.py -c <path to csv file to write locally> -j <path to rally json file> -a <atomic action(Eg.: nova.boot_server>
|
||||
-g -s <path to google service account json credentials file> -e <email id of user> -n <name of google sheet to be created>
|
||||
$ python rally_google_sheet_gen.py -c -f <path to directory to write csv file locally>
|
||||
-j <path to rally json file>
|
||||
-a <space separated list of atomic actions(Eg.: boot_server create_network)> -g
|
||||
-s <path to google service account json credentials file>
|
||||
-e <email id of user> -n <name of google sheet to be created>
|
||||
|
||||
@ -14,3 +14,5 @@ pykwalify>=1.8.0;python_version>='3'
|
||||
gspread
|
||||
oauth2client
|
||||
gspread_formatting
|
||||
pandas
|
||||
gspread_dataframe
|
||||
|
||||
@ -20,57 +20,78 @@
|
||||
# The script sends an email to the email id of the user with the Google sheet
|
||||
# if the --uploadgooglesheet option is enabled.
|
||||
|
||||
# To generate only a CSV file and not upload to Google Sheets :
|
||||
# To generate only CSV files and not upload to Google Sheets :
|
||||
|
||||
# $ source .browbeat-venv/bin/activate && cd utils
|
||||
# $ python rally_google_sheet_gen.py -c <path to csv file to write locally>
|
||||
# -j <path to rally json file> -a <atomic action(Eg.: nova.boot_server>
|
||||
# $ python rally_google_sheet_gen.py -c -f <path to directory to write csv file locally>
|
||||
# -j <path to rally json file>
|
||||
# -a <space separated list of atomic actions(Eg.: boot_server create_network)>
|
||||
|
||||
# To only upload to Google Sheets and not generate CSV files :
|
||||
|
||||
# $ source .browbeat-venv/bin/activate && cd utils
|
||||
# $ python rally_google_sheet_gen.py
|
||||
# -j <path to rally json file>
|
||||
# -a <space separated list of atomic actions(Eg.: boot_server create_network)> -g
|
||||
# -s <path to google service account json credentials file>
|
||||
# -e <email id of user> -n <name of google sheet to be created>
|
||||
|
||||
# To generate a CSV file and upload to Google Sheets :
|
||||
|
||||
# $ source .browbeat-venv/bin/activate && cd utils
|
||||
# $ python rally_google_sheet_gen.py -c <path to csv file to write locally>
|
||||
# -j <path to rally json file> -a <atomic action(Eg.: nova.boot_server> -g
|
||||
# $ python rally_google_sheet_gen.py -c -f <path to directory to write csv file locally>
|
||||
# -j <path to rally json file>
|
||||
# -a <space separated list of atomic actions(Eg.: boot_server create_network)> -g
|
||||
# -s <path to google service account json credentials file>
|
||||
# -e <email id of user> -n <name of google sheet to be created>
|
||||
|
||||
import json
|
||||
import csv
|
||||
import pandas as pd
|
||||
import gspread
|
||||
from oauth2client.service_account import ServiceAccountCredentials
|
||||
from gspread_formatting import cellFormat
|
||||
from gspread_formatting import format_cell_range
|
||||
from gspread_formatting import set_column_width
|
||||
from gspread_dataframe import set_with_dataframe
|
||||
import argparse
|
||||
|
||||
def convert_rally_json_to_csv(json_file_path, atomic_action, csv_file_path):
|
||||
def add_atomic_action_data_to_df(json_file_path, atomic_action, df_dict):
|
||||
"""Convert rally json data to csv file containing atomic action data
|
||||
:param json_file_path: str, path to rally json file
|
||||
:param atomic_action: str, atomic action to generate duration data
|
||||
:param csv_file_path: str, path to write csv file
|
||||
:param df_dict: dict, dict of dataframes from different atomic actions
|
||||
"""
|
||||
json_file = open(json_file_path)
|
||||
json_data = json.load(json_file)
|
||||
|
||||
with open(csv_file_path, 'w', newline='') as csvfile:
|
||||
csvwriter = csv.writer(csvfile, delimiter=',',
|
||||
quotechar='|', quoting=csv.QUOTE_MINIMAL)
|
||||
df = pd.DataFrame({"Resource Number": [], "Duration(in seconds)": []})
|
||||
|
||||
csvwriter.writerow(["Resource Number", "Duration(in seconds)"])
|
||||
resource_number = 1
|
||||
resource_number = 1
|
||||
|
||||
for iteration in json_data[0]["result"]:
|
||||
for atomic_action_json in iteration["atomic_actions"]:
|
||||
if atomic_action in atomic_action_json:
|
||||
atomic_action_duration = iteration["atomic_actions"][atomic_action_json]
|
||||
csvwriter.writerow([resource_number, atomic_action_duration])
|
||||
resource_number += 1
|
||||
for iteration in json_data[0]["result"]:
|
||||
for atomic_action_json in iteration["atomic_actions"]:
|
||||
if atomic_action in atomic_action_json:
|
||||
atomic_action_duration = iteration["atomic_actions"][atomic_action_json]
|
||||
df = df.append({"Resource Number": resource_number,
|
||||
"Duration(in seconds)": atomic_action_duration},
|
||||
ignore_index=True)
|
||||
resource_number += 1
|
||||
|
||||
print("CSV file {} generated successfully.".format(csv_file_path))
|
||||
df_dict[atomic_action] = df
|
||||
print("Pandas DF for atomic action {} generated successfully.".format(atomic_action))
|
||||
|
||||
def push_to_gsheet(csv_file_path, google_svc_acc_key, email_id):
|
||||
def generate_csv_from_df(csv_files_path, df_dict):
|
||||
"""Generate csv files from pandas dataframe
|
||||
:param csv_files_path: str, path of directory to generate csv files
|
||||
:param df_dict: dict, dict of dataframes from different atomic actions
|
||||
"""
|
||||
for atomic_action in df_dict:
|
||||
df_dict[atomic_action].to_csv("{}/{}.csv".format(csv_files_path, atomic_action))
|
||||
print("{}/{}.csv created succesfully".format(csv_files_path, atomic_action))
|
||||
|
||||
def push_to_gsheet(df_dict, google_svc_acc_key, email_id):
|
||||
"""Push csv file to Google Sheets
|
||||
:param csv_file_path: str, path to csv file
|
||||
:param df_dict: dict, dict of dataframes from different atomic actions
|
||||
:param google_svc_acc_key: str, path to json credentials for google service account
|
||||
:param email_id: str, email id of user who will be given edit access to google sheet
|
||||
"""
|
||||
@ -85,36 +106,40 @@ def push_to_gsheet(csv_file_path, google_svc_acc_key, email_id):
|
||||
gc = gspread.authorize(credentials)
|
||||
|
||||
sh = gc.create(params.sheetname)
|
||||
|
||||
for atomic_action in df_dict:
|
||||
ws = sh.add_worksheet(atomic_action, rows=len(df_dict[atomic_action]), cols=2)
|
||||
set_with_dataframe(ws, df_dict[atomic_action])
|
||||
format_cell_range(ws, "1:1000", fmt)
|
||||
set_column_width(ws, "A", 290)
|
||||
set_column_width(ws, "B", 190)
|
||||
|
||||
sh.share(email_id, perm_type="user", role="writer")
|
||||
spreadsheet_id = sh.id
|
||||
spreadsheet_url = f"https://docs.google.com/spreadsheets/d/{sh.id}"
|
||||
with open(csv_file_path, "r") as f:
|
||||
gc.import_csv(spreadsheet_id, f.read())
|
||||
worksheet = sh.get_worksheet(0)
|
||||
format_cell_range(worksheet, "1:1000", fmt)
|
||||
set_column_width(worksheet, "A", 290)
|
||||
set_column_width(worksheet, "B", 190)
|
||||
set_column_width(worksheet, "C", 400)
|
||||
print(f"Google Spreadsheet link -> {spreadsheet_url}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-c", "--csvfilepath", help="Path to create CSV file",
|
||||
nargs='?', required=True, dest="csvfilepath"
|
||||
'-c', '--generatecsvfiles', action='store_true', required=False,
|
||||
help='Flag to enable csv file generation', dest="generatecsvfiles"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-f", "--csvfilespath", help="Path to directory to create CSV files",
|
||||
nargs='?', required=False, dest="csvfilespath"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-j", "--jsonfilepath", help="Rally JSON file path to scrape output from",
|
||||
nargs='?', required=True, dest="jsonfilepath",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-a", "--atomicaction", help="Atomic action to generate duration data",
|
||||
nargs='?', required=True, dest="atomicaction",
|
||||
"-a", "--atomicactions", help="Atomic actions to generate duration data",
|
||||
nargs='+', required=True, dest="atomicactions",
|
||||
)
|
||||
parser.add_argument(
|
||||
'-g', '--uploadtogooglesheets', action='store_true', required=False,
|
||||
help='Flag to enable csv file upload to Google Sheets', dest="uploadtogooglesheets"
|
||||
help='Flag to enable upload to Google Sheets', dest="uploadtogooglesheets"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-n", "--sheetname", help="Google Sheet name",
|
||||
@ -130,6 +155,12 @@ if __name__ == "__main__":
|
||||
)
|
||||
params = parser.parse_args()
|
||||
|
||||
if(params.generatecsvfiles and params.csvfilespath is None):
|
||||
parser.error("{} requires {}.".format("--generatecsvfiles",
|
||||
"--csvfilespath"))
|
||||
|
||||
df_dict = {}
|
||||
|
||||
if(params.uploadtogooglesheets and
|
||||
(params.sheetname is None or params.googlesvcacckey is None or params.emailid is None)):
|
||||
parser.error("{} requires {} and {} and {}.".format("--uploadtogooglesheets",
|
||||
@ -137,7 +168,12 @@ if __name__ == "__main__":
|
||||
"--googlesvcacckey",
|
||||
"--emailid"))
|
||||
|
||||
convert_rally_json_to_csv(params.jsonfilepath, params.atomicaction,
|
||||
params.csvfilepath)
|
||||
for atomic_action in params.atomicactions:
|
||||
add_atomic_action_data_to_df(params.jsonfilepath, atomic_action,
|
||||
df_dict)
|
||||
|
||||
if params.generatecsvfiles:
|
||||
generate_csv_from_df(params.csvfilespath, df_dict)
|
||||
|
||||
if params.uploadtogooglesheets:
|
||||
push_to_gsheet(params.csvfilepath, params.googlesvcacckey, params.emailid)
|
||||
push_to_gsheet(df_dict, params.googlesvcacckey, params.emailid)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user