diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py index ff518c2d4a..38c5eb6622 100644 --- a/release_scripts/localization_scripts/allbundlesscript.py +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -3,44 +3,21 @@ This script requires the python libraries: gitpython and jproperties. As a cons git >= 1.7.0 and python >= 3.4. This script relies on fetching 'HEAD' from current branch. So make sure repo is on correct branch (i.e. develop). """ -import collections import sys - from envutil import get_proj_dir -from excelutil import records_to_excel -from fileutil import get_filename_addition, OMITTED_ADDITION +from excelutil import write_results_to_xlsx from gitutil import get_property_file_entries, get_commit_id, get_git_root -from csvutil import records_to_csv -from typing import Union, TypedDict, List +from csvutil import write_results_to_csv +from typing import Union import re import argparse - -class ProcessingResult: - results: List[List[str]] - omitted: Union[List[List[str]], None] - - def __init__(self, results: List[List[str]], omitted: Union[List[List[str]], None]): - self.results = results - self.omitted = omitted - - -def write_items_to_csv(results: ProcessingResult, output_path: str): - records_to_csv(output_path, results.results) - if results.omitted: - records_to_csv(get_filename_addition(output_path, OMITTED_ADDITION), results.omitted) - - -def write_items_to_xlsx(results: ProcessingResult, output_path: str): - workbook = collections.OrderedDict([('results', results.results)]) - if results.omitted: - workbook['omitted'] = results.omitted - - records_to_excel(output_path, workbook) +from outputresult import OutputResult +from outputtype import OutputType def get_items_to_be_written(repo_path: str, show_commit: bool, - value_regex: Union[str, None] = None) -> ProcessingResult: + value_regex: Union[str, None] = None) -> OutputResult: """Determines the contents of '.properties-MERGED' files and writes to a csv file. Args: @@ -65,28 +42,37 @@ def get_items_to_be_written(repo_path: str, show_commit: bool, omitted.append(new_entry) omitted_to_write = [row_header] + omitted if len(omitted) > 0 else None - return ProcessingResult([row_header] + rows, omitted_to_write) + return OutputResult([row_header] + rows, omitted_to_write) def main(): # noinspection PyTypeChecker parser = argparse.ArgumentParser(description='Gathers all key-value pairs within .properties-MERGED files into ' - 'one csv file.', + 'one file.', formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument(dest='output_path', type=str, help='The path to the output csv file. The output path should be' + parser.add_argument(dest='output_path', type=str, help='The path to the output file. The output path should be' ' specified as a relative path with the dot slash notation ' - '(i.e. \'./outputpath.csv\') or an absolute path.') + '(i.e. \'./outputpath.xlsx\') or an absolute path.') parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, help='The path to the repo. If not specified, path of script is used.') + parser.add_argument('-o', '--output-type', dest='output_type', type=OutputType, choices=list(OutputType), + required=False, help="The output type. Currently supports 'csv' or 'xlsx'.", default='xlsx') parser.add_argument('-nc', '--no_commit', dest='no_commit', action='store_true', default=False, - required=False, help="Suppresses adding commits to the generated csv header.") + required=False, help="Suppresses adding commits to the generated header.") args = parser.parse_args() repo_path = args.repo_path if args.repo_path is not None else get_git_root(get_proj_dir()) output_path = args.output_path show_commit = not args.no_commit + output_type = args.output_type - write_items_to_csv(repo_path, output_path, show_commit) + processing_result = get_items_to_be_written(repo_path, show_commit) + + # based on https://stackoverflow.com/questions/60208/replacements-for-switch-statement-in-python + { + OutputType.csv: write_results_to_csv, + OutputType.xlsx: write_results_to_xlsx + }[output_type](processing_result, output_path) sys.exit(0) diff --git a/release_scripts/localization_scripts/csvutil.py b/release_scripts/localization_scripts/csvutil.py index e44fd6ec06..f0886b91fa 100644 --- a/release_scripts/localization_scripts/csvutil.py +++ b/release_scripts/localization_scripts/csvutil.py @@ -4,6 +4,9 @@ from typing import List, Iterable, Tuple import csv import os +from fileutil import OMITTED_ADDITION, get_filename_addition, DELETED_ADDITION +from outputresult import OutputResult + def records_to_csv(output_path: str, rows: Iterable[List[str]]): """Writes rows to a csv file at the specified path. @@ -49,3 +52,20 @@ def csv_to_records(input_path: str, header_row: bool) -> Tuple[List[List[str]], raise Exception("There was an error parsing csv {path}".format(path=input_path), e) return results, header + + +def write_results_to_csv(results: OutputResult, output_path: str): + """ + Writes the result of processing to the output path as a csv file. If omitted values are present, for output_path of + /dir/file.csv, the omitted will be written to /dir/file-omitted.csv. + + Args: + results: The results to be written. + output_path: The output path. + """ + + records_to_csv(output_path, results.results) + if results.omitted: + records_to_csv(get_filename_addition(output_path, OMITTED_ADDITION), results.omitted) + if results.deleted: + records_to_csv(get_filename_addition(output_path, DELETED_ADDITION), results.deleted) diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index 76ed8326f6..f7b95be0cd 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -2,47 +2,22 @@ and generates a csv file containing the items changed. This script requires the python libraries: gitpython and jproperties. As a consequence, it also requires git >= 1.7.0 and python >= 3.4. """ -import collections import re import sys from envutil import get_proj_dir -from excelutil import records_to_excel -from fileutil import get_filename_addition, OMITTED_ADDITION, DELETED_ADDITION +from excelutil import write_results_to_xlsx from gitutil import get_property_files_diff, get_commit_id, get_git_root from itemchange import ItemChange, ChangeType -from csvutil import records_to_csv +from csvutil import write_results_to_csv import argparse -from typing import Union, TypedDict, List +from typing import Union from langpropsutil import get_commit_for_language, LANG_FILENAME - - -class DiffResult: - results: List[List[str]] - deleted: Union[List[List[str]], None] - omitted: Union[List[List[str]], None] - - -def write_items_to_csv(results: DiffResult, output_path: str): - records_to_csv(output_path, results.results) - if results.deleted: - records_to_csv(get_filename_addition(output_path, DELETED_ADDITION), results.deleted) - if results.omitted: - records_to_csv(get_filename_addition(output_path, OMITTED_ADDITION), results.omitted) - - -def write_items_to_xlsx(results: DiffResult, output_path: str): - workbook = collections.OrderedDict([('results', results.results)]) - if results.deleted: - workbook['deleted'] = results.deleted - - if results.omitted: - workbook['omitted'] = results.omitted - - records_to_excel(output_path, workbook) +from outputresult import OutputResult +from outputtype import OutputType def get_diff_to_write(repo_path: str, commit_1_id: str, commit_2_id: str, show_commits: bool, separate_deleted: bool, - value_regex: Union[str, None] = None) -> DiffResult: + value_regex: Union[str, None] = None) -> OutputResult: """Determines the changes made in '.properties-MERGED' files from one commit to another commit and returns results. Args: @@ -66,7 +41,7 @@ def get_diff_to_write(repo_path: str, commit_1_id: str, commit_2_id: str, show_c for entry in get_property_files_diff(repo_path, commit_1_id, commit_2_id): entry_row = entry.get_row() - if entry.type == ChangeType.DELETION: + if separate_deleted and entry.type == ChangeType.DELETION: deleted.append(entry_row) if value_regex is not None and re.match(value_regex, entry.cur_val): omitted.append(entry_row) @@ -76,13 +51,7 @@ def get_diff_to_write(repo_path: str, commit_1_id: str, commit_2_id: str, show_c omitted_result = [row_header] + omitted if len(omitted) > 0 else None deleted_result = [row_header] + deleted if len(deleted) > 0 else None - return { - 'results': [row_header] + rows, - 'omitted': omitted_result, - 'deleted': deleted_result - } - - + return OutputResult([row_header] + rows, omitted_result, deleted_result) def main(): @@ -91,9 +60,9 @@ def main(): "'.properties-MERGED' files and generates a csv file containing " "the items changed.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument(dest='output_path', type=str, help='The path to the output csv file. The output path should ' + parser.add_argument(dest='output_path', type=str, help='The path to the output file. The output path should ' 'be specified as a relative path with the dot slash notation' - ' (i.e. \'./outputpath.csv\') or an absolute path.') + ' (i.e. \'./outputpath.xlsx\') or an absolute path.') parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, help='The path to the repo. If not specified, path of script is used.') @@ -104,6 +73,8 @@ def main(): help='The commit for current release.') parser.add_argument('-nc', '--no-commits', dest='no_commits', action='store_true', default=False, required=False, help="Suppresses adding commits to the generated csv header.") + parser.add_argument('-o', '--output-type', dest='output_type', type=OutputType, choices=list(OutputType), + required=False, help="The output type. Currently supports 'csv' or 'xlsx'.", default='xlsx') parser.add_argument('-l', '--language', dest='language', type=str, default=None, required=False, help='Specify the language in order to determine the first commit to use (i.e. \'ja\' for ' 'Japanese. This flag overrides the first-commit flag.') @@ -112,6 +83,8 @@ def main(): repo_path = args.repo_path if args.repo_path is not None else get_git_root(get_proj_dir()) output_path = args.output_path commit_1_id = args.commit_1_id + output_type = args.output_type + lang = args.language if lang is not None: commit_1_id = get_commit_for_language(lang) @@ -125,7 +98,13 @@ def main(): commit_2_id = args.commit_2_id show_commits = not args.no_commits - write_diff_to_csv(repo_path, output_path, commit_1_id, commit_2_id, show_commits) + processing_result = get_diff_to_write(repo_path, commit_1_id, commit_2_id, show_commits) + + # based on https://stackoverflow.com/questions/60208/replacements-for-switch-statement-in-python + { + OutputType.csv: write_results_to_csv, + OutputType.xlsx: write_results_to_xlsx + }[output_type](processing_result, output_path) sys.exit(0) diff --git a/release_scripts/localization_scripts/excelutil.py b/release_scripts/localization_scripts/excelutil.py index 95788e820f..e114b0ee52 100644 --- a/release_scripts/localization_scripts/excelutil.py +++ b/release_scripts/localization_scripts/excelutil.py @@ -1,10 +1,22 @@ """Provides tools for parsing and writing to a csv file. """ -from typing import List, Dict, OrderedDict +import collections +from typing import List, OrderedDict import pyexcel +from outputresult import OutputResult + Workbook = OrderedDict[str, List[List[str]]] +# The name for the results sheet +RESULTS_SHEET_NAME = 'results' + +# The name for the sheet of deleted items +DELETED_SHEET_NAME = 'deleted' + +# The name for the sheet of omitted items +OMITTED_SHEET_NAME = 'omitted' + def records_to_excel(output_path: str, workbook: Workbook): """Writes workbook data model to an excel workbook. @@ -22,13 +34,32 @@ def records_to_excel(output_path: str, workbook: Workbook): ) -def csv_to_records(input_path: str) -> Workbook: - """Reads rows to a csv file at the specified path. +def excel_to_records(input_path: str) -> Workbook: + """Reads rows to a excel file at the specified path. Args: - input_path (str): The path where the csv file will be written. + input_path (str): The path where the excel file will be read. """ return pyexcel.get_book_dict( file_name=input_path - ) \ No newline at end of file + ) + + +def write_results_to_xlsx(results: OutputResult, output_path: str): + """ + Writes the result of processing to the output path as a xlsx file. Results will be written to a 'results' sheet. + Omitted results will be written to an 'omitted' sheet. Deleted results will be written to a 'deleted' sheet. + + Args: + results: The results to be written. + output_path: The output path. + """ + + workbook = collections.OrderedDict([(RESULTS_SHEET_NAME, results.results)]) + if results.omitted: + workbook[OMITTED_SHEET_NAME] = results.omitted + if results.deleted: + workbook[DELETED_SHEET_NAME] = results.deleted + + records_to_excel(output_path, workbook) diff --git a/release_scripts/localization_scripts/fileutil.py b/release_scripts/localization_scripts/fileutil.py index 3e5565d9ff..22b1f5ef97 100644 --- a/release_scripts/localization_scripts/fileutil.py +++ b/release_scripts/localization_scripts/fileutil.py @@ -44,7 +44,8 @@ def get_new_path(orig_path: str, new_filename: str) -> str: OMITTED_ADDITION = '-omitted' # For use with creating csv filenames for entries that have been deleted. -OMITTED_ADDITION = '-deleted' +DELETED_ADDITION = '-deleted' + def get_filename_addition(orig_path: str, filename_addition: str) -> str: """Gets filename with addition. So if item is '/path/name.ext' and the filename_addition is '-add', the new result diff --git a/release_scripts/localization_scripts/outputresult.py b/release_scripts/localization_scripts/outputresult.py new file mode 100644 index 0000000000..60ea6cd63d --- /dev/null +++ b/release_scripts/localization_scripts/outputresult.py @@ -0,0 +1,28 @@ +from typing import List, Union + + +class OutputResult: + """ + Describes a result that is ready to be written to file(s). + """ + + results: List[List[str]] + omitted: Union[List[List[str]], None] + deleted: Union[List[List[str]], None] + + def __init__(self, results: List[List[str]], omitted: Union[List[List[str]], None] = None, + deleted: Union[List[List[str]], None] = None): + """ + Constructs a ProcessingResult. + Args: + results: Items to be written as results. Data will be written such that the item at row,cell will be + located within result at results[row][col]. + omitted: Items to be written as omitted. Data will be written such that the item at row,cell will be + located within result at results[row][col]. + deleted: Items to be written as omitted. Data will be written such that the item at row,cell will be + located within result at results[row][col]. + """ + + self.results = results + self.omitted = omitted + self.deleted = deleted diff --git a/release_scripts/localization_scripts/outputtype.py b/release_scripts/localization_scripts/outputtype.py new file mode 100644 index 0000000000..8b488df506 --- /dev/null +++ b/release_scripts/localization_scripts/outputtype.py @@ -0,0 +1,10 @@ +from enum import Enum + + +class OutputType(Enum): + """Describes the output file type.""" + xlsx = 'xlsx' + csv = 'csv' + + def __str__(self): + return str(self.value) \ No newline at end of file