From ae1b435bb26e50262311cdf6a03fd2ce77fac055 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 4 Jan 2021 16:55:23 -0500 Subject: [PATCH] initial commit --- .../localization_scripts/csvutil.py | 4 +++- .../localization_scripts/diffscript.py | 4 ++++ .../localization_scripts/excelutil.py | 5 +++++ .../localization_scripts/fileutil.py | 16 +++++++++++++++ .../localization_scripts/itemchange.py | 20 ++++++++++++++----- .../localization_scripts/outputresult.py | 10 +++++++--- .../localization_scripts/tabularutil.py | 3 +++ 7 files changed, 53 insertions(+), 9 deletions(-) diff --git a/release_scripts/localization_scripts/csvutil.py b/release_scripts/localization_scripts/csvutil.py index f0886b91fa..e34dc8ea17 100644 --- a/release_scripts/localization_scripts/csvutil.py +++ b/release_scripts/localization_scripts/csvutil.py @@ -4,7 +4,7 @@ from typing import List, Iterable, Tuple import csv import os -from fileutil import OMITTED_ADDITION, get_filename_addition, DELETED_ADDITION +from fileutil import OMITTED_ADDITION, get_filename_addition, DELETED_ADDITION, FOUND_ADDITION from outputresult import OutputResult @@ -69,3 +69,5 @@ def write_results_to_csv(results: OutputResult, output_path: str): records_to_csv(get_filename_addition(output_path, OMITTED_ADDITION), results.omitted) if results.deleted: records_to_csv(get_filename_addition(output_path, DELETED_ADDITION), results.deleted) + if results.found: + records_to_csv(get_filename_addition(output_path, FOUND_ADDITION), results.found) diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index c2dc154b65..c1b04af152 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -41,6 +41,10 @@ def main(): help='Specify the path to the properties file containing key value pairs of language mapped to ' 'the commit of when bundles for that language were most recently updated.') + parser.add_argument('-t', '--translation-dict', dest='translation_dict', type=bool, required=False, default=False, + help='If this flag is specified, a dictionary mapping original prop key values to translated ' + 'values. If this flag is specified, it will ') + parser.add_argument('-nt', '--no-translated-col', dest='no_translated_col', action='store_true', default=False, required=False, help="Don't include a column for translation.") diff --git a/release_scripts/localization_scripts/excelutil.py b/release_scripts/localization_scripts/excelutil.py index 460006ec16..66c51f7099 100644 --- a/release_scripts/localization_scripts/excelutil.py +++ b/release_scripts/localization_scripts/excelutil.py @@ -19,6 +19,9 @@ DELETED_SHEET_NAME = 'deleted' # The name for the sheet of omitted items OMITTED_SHEET_NAME = 'omitted' +# The name for the sheet of found items +FOUND_SHEET_NAME = 'found' + def excel_to_records(input_path: str) -> Workbook: """Reads rows to a excel file at the specified path. @@ -55,6 +58,8 @@ def write_results_to_xlsx(results: OutputResult, output_path: str): workbook[OMITTED_SHEET_NAME] = results.omitted if results.deleted: workbook[DELETED_SHEET_NAME] = results.deleted + if results.found: + workbook[FOUND_SHEET_NAME] = results.found wb_file = xlsxwriter.Workbook(output_path) styles = [] diff --git a/release_scripts/localization_scripts/fileutil.py b/release_scripts/localization_scripts/fileutil.py index 22b1f5ef97..561420220e 100644 --- a/release_scripts/localization_scripts/fileutil.py +++ b/release_scripts/localization_scripts/fileutil.py @@ -24,6 +24,19 @@ def get_path_pieces(orig_path: str) -> Tuple[str, Union[str, None], Union[str, N return potential_parent_dir, filename, file_extension +def get_joined_path(folder: str, file_name: str) -> str: + """ + Gets a joined folder and filename. + Args: + folder: The folder. + file_name: The filename. + + Returns: The joined string path. + + """ + return str(Path(folder) / Path(file_name)) + + def get_new_path(orig_path: str, new_filename: str) -> str: """Obtains a new path. This tries to determine if the provided path is a directory or filename (has an extension containing '.') then constructs the new path with the old parent directory and the new filename. @@ -46,6 +59,9 @@ OMITTED_ADDITION = '-omitted' # For use with creating csv filenames for entries that have been deleted. DELETED_ADDITION = '-deleted' +# For translations where +FOUND_ADDITION = '-found' + def get_filename_addition(orig_path: str, filename_addition: str) -> str: """Gets filename with addition. So if item is '/path/name.ext' and the filename_addition is '-add', the new result diff --git a/release_scripts/localization_scripts/itemchange.py b/release_scripts/localization_scripts/itemchange.py index 61a6cab941..54213dc98b 100644 --- a/release_scripts/localization_scripts/itemchange.py +++ b/release_scripts/localization_scripts/itemchange.py @@ -1,5 +1,6 @@ -from typing import Iterator, List, Union +from typing import Iterator, List, Union, Dict +from languagedictutil import FoundValue from outputresult import OutputResult from propsutil import get_entry_dict from enum import Enum @@ -82,15 +83,20 @@ class ItemChange: ITEMCHANGE_DEFAULT_COLS = [RELATIVE_PATH_COL, KEY_COL, 'Change Type', 'Previous Value', 'Current Value'] -def convert_to_output(items: Iterator[ItemChange], commit1_id: Union[str, None] = None, - commit2_id: Union[str, None] = None, show_translated_col: bool = True, - value_regex: Union[str, None] = None, separate_deleted: bool = True) -> OutputResult: +def convert_to_output(items: Iterator[ItemChange], + commit1_id: Union[str, None] = None, + commit2_id: Union[str, None] = None, + translation_dict: Union[Dict[str, FoundValue], None] = None, + show_translated_col: bool = True, + value_regex: Union[str, None] = None, + separate_deleted: bool = True) -> OutputResult: """ Converts PropEntry objects to an output result to be written to a tabular datasource. Args: items: The PropEntry items. commit1_id: The first commit id to be shown in the header or None. commit2_id: The second commit id to be shown in the header or None. + translation_dict: A dictionary of English values mapped to the translated value. show_translated_col: Whether or not to show an empty translated column. value_regex: Regex to determine if a value should be omitted. separate_deleted: Deleted items should not be included in regular results. @@ -109,6 +115,7 @@ def convert_to_output(items: Iterator[ItemChange], commit1_id: Union[str, None] results = [] omitted = [] + found_translation = [] deleted = [] for item in items: @@ -117,10 +124,13 @@ def convert_to_output(items: Iterator[ItemChange], commit1_id: Union[str, None] deleted.append(item_row) elif value_regex is not None and re.match(value_regex, item.cur_val): omitted.append(item_row) + elif translation_dict is not None and item.cur_val.strip() in translation_dict: + found_translation.append(item_row + [translation_dict[item.cur_val.strim()].translated_val]) else: results.append(item_row) - return create_output_result(header, results, omitted=omitted, deleted=deleted, style=style) + return create_output_result(header, results, omitted=omitted, deleted=deleted, found_translation=found_translation, + style=style) def get_item_change(rel_path: str, key: str, prev_val: str, cur_val: str) -> Union[ItemChange, None]: diff --git a/release_scripts/localization_scripts/outputresult.py b/release_scripts/localization_scripts/outputresult.py index a5169eede1..0fc9a55ac0 100644 --- a/release_scripts/localization_scripts/outputresult.py +++ b/release_scripts/localization_scripts/outputresult.py @@ -11,17 +11,18 @@ class ColumnStyle(TypedDict): class OutputResult: """ - Describes a result that is ready to be written to file(s). + Describes a result that is ready to be written to tabular file(s). """ column_styles: List[ColumnStyle] freeze_first_row: bool results: List[List[str]] omitted: Union[List[List[str]], None] deleted: Union[List[List[str]], None] + found: Union[List[List[str]], None] def __init__(self, results: List[List[str]], omitted: Union[List[List[str]], None] = None, - deleted: Union[List[List[str]], None] = None, style: Union[List[ColumnStyle], None] = None, - freeze_first_row: bool = True): + deleted: Union[List[List[str]], None] = None, found: Union[List[List[str]], None] = None, + style: Union[List[ColumnStyle], None] = None, freeze_first_row: bool = True): """ Constructs a ProcessingResult. Args: @@ -31,6 +32,8 @@ class OutputResult: located within result at results[row][col]. deleted: Items to be written as omitted. Data will be written such that the item at row,cell will be located within result at results[row][col]. + found: Items where a translation was found elsewhere. Data will be written such that the item at row,cell + will be located within result at results[row][col]. style: Style for each column. No column formatting will happen for null. freeze_first_row: Whether or not first row should be frozen. """ @@ -38,5 +41,6 @@ class OutputResult: self.results = results self.omitted = omitted self.deleted = deleted + self.found = found self.column_styles = style self.freeze_first_row = freeze_first_row diff --git a/release_scripts/localization_scripts/tabularutil.py b/release_scripts/localization_scripts/tabularutil.py index 3a05f1ee91..4862cb2e38 100644 --- a/release_scripts/localization_scripts/tabularutil.py +++ b/release_scripts/localization_scripts/tabularutil.py @@ -27,6 +27,7 @@ WITH_TRANSLATED_STYLE = [None, None, VALUE_STYLE, VALUE_STYLE] def create_output_result(row_header: List[str], results: List[List[str]], omitted: Union[List[List[str]], None] = None, deleted: Union[List[List[str]], None] = None, + found_translation: Union[List[List[str]], None] = None, style: Union[List[ColumnStyle], None] = None) -> OutputResult: """ @@ -36,6 +37,7 @@ def create_output_result(row_header: List[str], results: List[List[str]], results: The results. omitted: The omitted items if any. deleted: The deleted items if any. + found_translation: Items where a previous translation has already been created. style: Style of columns if any. Returns: The generated OutputResult. @@ -43,5 +45,6 @@ def create_output_result(row_header: List[str], results: List[List[str]], """ omitted_result = [row_header] + omitted if omitted else None deleted_result = [row_header] + deleted if deleted else None + found_result = [row_header] + found_translation if found_translation else None return OutputResult([row_header] + results, omitted_result, deleted_result, style)