diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py index 0bcb93fde5..ff518c2d4a 100644 --- a/release_scripts/localization_scripts/allbundlesscript.py +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -3,10 +3,11 @@ This script requires the python libraries: gitpython and jproperties. As a cons git >= 1.7.0 and python >= 3.4. This script relies on fetching 'HEAD' from current branch. So make sure repo is on correct branch (i.e. develop). """ - +import collections import sys from envutil import get_proj_dir +from excelutil import records_to_excel from fileutil import get_filename_addition, OMITTED_ADDITION from gitutil import get_property_file_entries, get_commit_id, get_git_root from csvutil import records_to_csv @@ -15,31 +16,31 @@ import re import argparse -class ProcessingResult(TypedDict): +class ProcessingResult: results: List[List[str]] - omitted: List[List[str]] + omitted: Union[List[List[str]], None] + + def __init__(self, results: List[List[str]], omitted: Union[List[List[str]], None]): + self.results = results + self.omitted = omitted - -def write_items_to_csv(repo_path: str, output_path: str, show_commit: bool, value_regex: Union[str, None] = None): - """Determines the contents of '.properties-MERGED' files and writes to a csv file. - - Args: - repo_path (str): The local path to the git repo. - output_path (str): The output path for the csv file. - show_commit (bool): Whether or not to include the commit id in the header - value_regex (Union[str, None]): If non-none, only key value pairs where the value is a regex match with this - value will be included. - """ - pass - # records_to_csv(output_path, [row_header] + rows) - # - # if len(omitted) > 0: - # records_to_csv(get_filename_addition(output_path, OMITTED_ADDITION), [row_header] + omitted) +def write_items_to_csv(results: ProcessingResult, output_path: str): + records_to_csv(output_path, results.results) + if results.omitted: + records_to_csv(get_filename_addition(output_path, OMITTED_ADDITION), results.omitted) +def write_items_to_xlsx(results: ProcessingResult, output_path: str): + workbook = collections.OrderedDict([('results', results.results)]) + if results.omitted: + workbook['omitted'] = results.omitted -def get_items_to_be_written(repo_path: str, show_commit: bool, value_regex: Union[str, None] = None) -> ProcessingResult: + records_to_excel(output_path, workbook) + + +def get_items_to_be_written(repo_path: str, show_commit: bool, + value_regex: Union[str, None] = None) -> ProcessingResult: """Determines the contents of '.properties-MERGED' files and writes to a csv file. Args: @@ -63,7 +64,8 @@ def get_items_to_be_written(repo_path: str, show_commit: bool, value_regex: Unio else: omitted.append(new_entry) - return {'results': rows, 'omitted': omitted} + omitted_to_write = [row_header] + omitted if len(omitted) > 0 else None + return ProcessingResult([row_header] + rows, omitted_to_write) def main(): diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index 2713fef518..76ed8326f6 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -2,28 +2,56 @@ and generates a csv file containing the items changed. This script requires the python libraries: gitpython and jproperties. As a consequence, it also requires git >= 1.7.0 and python >= 3.4. """ +import collections import re import sys from envutil import get_proj_dir -from fileutil import get_filename_addition, OMITTED_ADDITION +from excelutil import records_to_excel +from fileutil import get_filename_addition, OMITTED_ADDITION, DELETED_ADDITION from gitutil import get_property_files_diff, get_commit_id, get_git_root from itemchange import ItemChange, ChangeType from csvutil import records_to_csv import argparse -from typing import Union +from typing import Union, TypedDict, List from langpropsutil import get_commit_for_language, LANG_FILENAME -def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit_2_id: str, show_commits: bool, - value_regex: Union[str, None] = None): - """Determines the changes made in '.properties-MERGED' files from one commit to another commit. +class DiffResult: + results: List[List[str]] + deleted: Union[List[List[str]], None] + omitted: Union[List[List[str]], None] + + +def write_items_to_csv(results: DiffResult, output_path: str): + records_to_csv(output_path, results.results) + if results.deleted: + records_to_csv(get_filename_addition(output_path, DELETED_ADDITION), results.deleted) + if results.omitted: + records_to_csv(get_filename_addition(output_path, OMITTED_ADDITION), results.omitted) + + +def write_items_to_xlsx(results: DiffResult, output_path: str): + workbook = collections.OrderedDict([('results', results.results)]) + if results.deleted: + workbook['deleted'] = results.deleted + + if results.omitted: + workbook['omitted'] = results.omitted + + records_to_excel(output_path, workbook) + + +def get_diff_to_write(repo_path: str, commit_1_id: str, commit_2_id: str, show_commits: bool, separate_deleted: bool, + value_regex: Union[str, None] = None) -> DiffResult: + """Determines the changes made in '.properties-MERGED' files from one commit to another commit and returns results. Args: repo_path (str): The local path to the git repo. - output_path (str): The output path for the csv file. commit_1_id (str): The initial commit for the diff. commit_2_id (str): The latest commit for the diff. show_commits (bool): Show commits in the header row. + separate_deleted (bool): put deletion items in a separate field in return type ('deleted'). Otherwise, + include in regular results. value_regex (Union[str, None]): If non-none, only key value pairs where the value is a regex match with this value will be included. """ @@ -34,18 +62,27 @@ def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit rows = [] omitted = [] + deleted = [] for entry in get_property_files_diff(repo_path, commit_1_id, commit_2_id): - new_entry = entry.get_row() - if value_regex is not None and (entry.type == ChangeType.DELETION or not re.match(value_regex, entry.cur_val)): - omitted.append(new_entry) + entry_row = entry.get_row() + if entry.type == ChangeType.DELETION: + deleted.append(entry_row) + if value_regex is not None and re.match(value_regex, entry.cur_val): + omitted.append(entry_row) else: - rows.append(new_entry) + rows.append(entry_row) + + omitted_result = [row_header] + omitted if len(omitted) > 0 else None + deleted_result = [row_header] + deleted if len(deleted) > 0 else None + + return { + 'results': [row_header] + rows, + 'omitted': omitted_result, + 'deleted': deleted_result + } - records_to_csv(output_path, [row_header] + rows) - if len(omitted) > 0: - records_to_csv(get_filename_addition(output_path, OMITTED_ADDITION), [row_header] + omitted) def main(): diff --git a/release_scripts/localization_scripts/excelutil.py b/release_scripts/localization_scripts/excelutil.py index deae929f7a..95788e820f 100644 --- a/release_scripts/localization_scripts/excelutil.py +++ b/release_scripts/localization_scripts/excelutil.py @@ -1,9 +1,9 @@ """Provides tools for parsing and writing to a csv file. """ -from typing import List, Dict +from typing import List, Dict, OrderedDict import pyexcel -Workbook = Dict[str, List[List[str]]] +Workbook = OrderedDict[str, List[List[str]]] def records_to_excel(output_path: str, workbook: Workbook): diff --git a/release_scripts/localization_scripts/fileutil.py b/release_scripts/localization_scripts/fileutil.py index 5139812db2..3e5565d9ff 100644 --- a/release_scripts/localization_scripts/fileutil.py +++ b/release_scripts/localization_scripts/fileutil.py @@ -43,6 +43,8 @@ def get_new_path(orig_path: str, new_filename: str) -> str: # For use with creating csv filenames for entries that have been omitted. OMITTED_ADDITION = '-omitted' +# For use with creating csv filenames for entries that have been deleted. +OMITTED_ADDITION = '-deleted' def get_filename_addition(orig_path: str, filename_addition: str) -> str: """Gets filename with addition. So if item is '/path/name.ext' and the filename_addition is '-add', the new result