diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index 3f08747f6e..270a8e609a 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -5,7 +5,7 @@ As a consequence, it also requires git >= 1.7.0 and python >= 3.4. import sys from envutil import get_proj_dir from excelutil import write_results_to_xlsx -from gitutil import get_property_files_diff, get_git_root, get_commit_id, get_tree +from gitutil import get_property_files_diff, get_git_root, get_commit_id, get_tree, list_paths from itemchange import convert_to_output from csvutil import write_results_to_csv import argparse @@ -43,7 +43,7 @@ def main(): help='Specify the path to the properties file containing key value pairs of language mapped to ' 'the commit of when bundles for that language were most recently updated.') - parser.add_argument('-td', '--translation-dict', dest='translation_dict', type=bool, required=False, default=False, + parser.add_argument('-td', '--translation-dict', dest='translation_dict', action='store_true', required=False, help='If this flag is specified, a dictionary mapping original prop key values to translated ' 'values. If this flag is specified, it will ') @@ -71,7 +71,7 @@ def main(): translation_dict = None if use_translation_dict and lang: translation_dict = extract_translations( - file_iter=get_tree(repo_path, commit_1_id), + file_iter=list_paths(get_tree(repo_path, commit_1_id)), orig_filename=DEFAULT_PROPS_FILENAME, translated_filename=get_lang_bundle_name(lang)) diff --git a/release_scripts/localization_scripts/foundvalue.py b/release_scripts/localization_scripts/foundvalue.py new file mode 100644 index 0000000000..cf61bf9ae7 --- /dev/null +++ b/release_scripts/localization_scripts/foundvalue.py @@ -0,0 +1,29 @@ +class FoundValue: + """ + A class containing a record of a prop key existing in both an original props file and a translated props file + """ + common_path: str + original_file: str + translated_file: str + key: str + orig_val: str + translated_val: str + + def __init__(self, common_path, original_file, translated_file, key, orig_val, translated_val): + """ + Constructor. + + Args: + common_path: The folder common to both files. + original_file: The original file path. + translated_file: The translated file path. + key: The common prop key. + orig_val: The original (English) value. + translated_val: The translated value. + """ + self.common_path = common_path + self.original_file = original_file + self.translated_file = translated_file + self.key = key + self.orig_val = orig_val + self.translated_val = translated_val diff --git a/release_scripts/localization_scripts/gitutil.py b/release_scripts/localization_scripts/gitutil.py index 54bdefcbad..fd4150cc89 100644 --- a/release_scripts/localization_scripts/gitutil.py +++ b/release_scripts/localization_scripts/gitutil.py @@ -128,7 +128,7 @@ def get_property_files_diff(repo_path: str, commit_1_id: str, commit_2_id: str, yield from get_changed_from_diff(rel_path, diff) -def list_paths(root_tree, path: Path = Path('.')) -> Iterator[Tuple[str, Blob]]: +def list_paths(root_tree: Tree, path: Path = Path('.')) -> Iterator[Tuple[str, Blob]]: """ Given the root path to serve as a prefix, walks the tree of a git commit returning all files and blobs. Repurposed from: https://www.enricozini.org/blog/2019/debian/gitpython-list-all-files-in-a-git-commit/ @@ -174,7 +174,7 @@ def get_property_file_entries(repo_path: str, at_commit: str = 'HEAD', Returns: An iterator of PropEntry objects. """ - for item in get_tree(repo_path, at_commit): + for item in list_paths(get_tree(repo_path, at_commit)): path, blob = item if path.endswith(property_file_extension): for key, val in get_entry_dict(get_text(blob)).items(): diff --git a/release_scripts/localization_scripts/itemchange.py b/release_scripts/localization_scripts/itemchange.py index dff8731ec3..198a5f4bad 100644 --- a/release_scripts/localization_scripts/itemchange.py +++ b/release_scripts/localization_scripts/itemchange.py @@ -1,6 +1,6 @@ from typing import Iterator, List, Union, Dict -from languagedictutil import FoundValue +from foundvalue import FoundValue from outputresult import OutputResult from propsutil import get_entry_dict from enum import Enum @@ -127,7 +127,7 @@ def convert_to_output(items: Iterator[ItemChange], elif value_regex is not None and re.match(value_regex, item.cur_val): omitted.append(item_row) elif translation_dict is not None and item.cur_val.strip() in translation_dict: - found_translation.append(item_row + [translation_dict[item.cur_val.strim()].translated_val]) + found_translation.append(item_row + [translation_dict[item.cur_val.strip()].translated_val]) else: results.append(item_row) diff --git a/release_scripts/localization_scripts/languagedictutil.py b/release_scripts/localization_scripts/languagedictutil.py index 9f45c4c8d1..9a18b8374e 100644 --- a/release_scripts/localization_scripts/languagedictutil.py +++ b/release_scripts/localization_scripts/languagedictutil.py @@ -2,40 +2,12 @@ import os from pathlib import Path from typing import Dict, Iterator, Tuple, TypeVar from git import Blob + +from foundvalue import FoundValue +from gitutil import get_text from propsutil import get_entry_dict -class FoundValue: - """ - A class containing a record of a prop key existing in both an original props file and a translated props file - """ - common_path: str - original_file: str - translated_file: str - key: str - orig_val: str - translated_val: str - - def __init__(self, common_path, original_file, translated_file, key, orig_val, translated_val): - """ - Constructor. - - Args: - common_path: The folder common to both files. - original_file: The original file path. - translated_file: The translated file path. - key: The common prop key. - orig_val: The original (English) value. - translated_val: The translated value. - """ - self.common_path = common_path - self.original_file = original_file - self.translated_file = translated_file - self.key = key - self.orig_val = orig_val - self.translated_val = translated_val - - def extract_translations(file_iter: Iterator[Tuple[str, Blob]], orig_filename: str, translated_filename: str) \ -> Dict[str, FoundValue]: """ @@ -61,18 +33,19 @@ def extract_translations(file_iter: Iterator[Tuple[str, Blob]], orig_filename: s for path, content in file_iter: parent_dir, file_name = os.path.split(str(Path(path))) if file_name.strip().lower() == orig_filename.strip().lower(): - original_files[file_name] = (parent_dir, content) + original_files[parent_dir.strip().lower()] = (path, content) elif file_name.strip().lower() == translated_filename.strip().lower(): - translated_files[file_name] = (parent_dir, content) + translated_files[parent_dir.strip().lower()] = (path, content) # determine original and translated files with common parent folders and find common keys to_ret: Dict[str, FoundValue] = dict() - for common_folder, ((original_path, original_blob), (translated_path, translated_blob))\ + for (common_folder, (original_path, original_blob), (translated_path, translated_blob))\ in common_entries(original_files, translated_files): - orig_dict = sanitize_prop_dict_keys(get_entry_dict(original_blob)) - translated_dict = sanitize_prop_dict_keys(get_entry_dict(translated_blob)) - for common_key, (orig_value, translated_value) in common_entries(orig_dict, translated_dict): + orig_dict = sanitize_prop_dict_keys(get_entry_dict(get_text(original_blob))) + translated_dict = sanitize_prop_dict_keys(get_entry_dict(get_text(translated_blob))) + + for common_key, orig_value, translated_value in common_entries(orig_dict, translated_dict): to_ret[orig_value] = FoundValue( common_path=common_folder, original_file=original_path,