From 202721acae8155da3214e03438726979c9531e21 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 10 Jul 2020 13:02:45 -0400 Subject: [PATCH 01/42] adding localization helper scripts --- release_scripts/diffscript.py | 220 ---------------- .../localization_scripts/.gitignore | 2 + .../localization_scripts/allbundlesscript.py | 43 ++++ .../localization_scripts/csvutil.py | 47 ++++ .../localization_scripts/diffscript.py | 61 +++++ .../localization_scripts/gitutil.py | 147 +++++++++++ .../localization_scripts/itemchange.py | 96 +++++++ .../localization_scripts/propentry.py | 14 + .../localization_scripts/propsutil.py | 65 +++++ .../localization_scripts/updatepropsscript.py | 240 ++++++++++++++++++ 10 files changed, 715 insertions(+), 220 deletions(-) delete mode 100644 release_scripts/diffscript.py create mode 100644 release_scripts/localization_scripts/.gitignore create mode 100644 release_scripts/localization_scripts/allbundlesscript.py create mode 100644 release_scripts/localization_scripts/csvutil.py create mode 100644 release_scripts/localization_scripts/diffscript.py create mode 100644 release_scripts/localization_scripts/gitutil.py create mode 100644 release_scripts/localization_scripts/itemchange.py create mode 100644 release_scripts/localization_scripts/propentry.py create mode 100644 release_scripts/localization_scripts/propsutil.py create mode 100644 release_scripts/localization_scripts/updatepropsscript.py diff --git a/release_scripts/diffscript.py b/release_scripts/diffscript.py deleted file mode 100644 index af3d995a88..0000000000 --- a/release_scripts/diffscript.py +++ /dev/null @@ -1,220 +0,0 @@ -"""This script determines the updated, added, and deleted properties from the '.properties-MERGED' files -and generates a csv file containing the items changed. This script requires the python libraries: -gitpython and jproperties. As a consequence, it also requires git >= 1.7.0 and python >= 3.4. -""" - -from git import Repo -from typing import List, Dict, Tuple -import re -import csv -from jproperties import Properties -import sys - - -class ItemChange: - def __init__(self, rel_path: str, key: str, prev_val: str, cur_val: str): - """Describes the change that occurred for a particular key of a properties file. - - Args: - rel_path (str): The relative path of the properties file. - key (str): The key in the properties file. - prev_val (str): The previous value for the key. - cur_val (str): The current value for the key. - """ - self.rel_path = rel_path - self.key = key - self.prev_val = prev_val - self.cur_val = cur_val - if ItemChange.has_str_content(cur_val) and not ItemChange.has_str_content(prev_val): - self.type = 'ADDITION' - elif not ItemChange.has_str_content(cur_val) and ItemChange.has_str_content(prev_val): - self.type = 'DELETION' - else: - self.type = 'CHANGE' - - @staticmethod - def has_str_content(content: str): - """Determines whether or not the content is empty or None. - - Args: - content (str): The text. - - Returns: - bool: Whether or not it has content. - """ - return content is not None and len(content.strip()) > 0 - - @staticmethod - def get_headers() -> List[str]: - """Returns the csv headers to insert when serializing a list of ItemChange objects to csv. - - Returns: - List[str]: The column headers - """ - return ['Relative Path', 'Key', 'Change Type', 'Previous Value', 'Current Value'] - - def get_row(self) -> List[str]: - """Returns the list of values to be entered as a row in csv serialization. - - Returns: - List[str]: The list of values to be entered as a row in csv serialization. - """ - return [ - self.rel_path, - self.key, - self.type, - self.prev_val, - self.cur_val] - - -def get_entry_dict(diff_str: str) -> Dict[str, str]: - """Retrieves a dictionary mapping the properties represented in the string. - - Args: - diff_str (str): The string of the properties file. - - Returns: - Dict[str,str]: The mapping of keys to values in that properties file. - """ - props = Properties() - props.load(diff_str, "utf-8") - return props.properties - - -def get_item_change(rel_path: str, key: str, prev_val: str, cur_val: str) -> ItemChange: - """Returns an ItemChange object if the previous value is not equal to the current value. - - Args: - rel_path (str): The relative path for the properties file. - key (str): The key within the properties file for this potential change. - prev_val (str): The previous value. - cur_val (str): The current value. - - Returns: - ItemChange: The ItemChange object or None if values are the same. - """ - if (prev_val == cur_val): - return None - else: - return ItemChange(rel_path, key, prev_val, cur_val) - - -def get_changed(rel_path: str, a_str: str, b_str: str) -> List[ItemChange]: - """Given the relative path of the properties file that - - Args: - rel_path (str): The relative path for the properties file. - a_str (str): The string representing the original state of the file. - b_str (str): The string representing the current state of the file. - - Returns: - List[ItemChange]: The changes determined. - """ - print('Retrieving changes for {}...'.format(rel_path)) - a_dict = get_entry_dict(a_str) - b_dict = get_entry_dict(b_str) - all_keys = set().union(a_dict.keys(), b_dict.keys()) - mapped = map(lambda key: get_item_change( - rel_path, key, a_dict.get(key), b_dict.get(key)), all_keys) - return filter(lambda entry: entry is not None, mapped) - - -def get_text(blob) -> str: - return blob.data_stream.read().decode('utf-8') - - -def get_changed_from_diff(rel_path: str, diff) -> List[ItemChange]: - """Determines changes from a git python diff. - - Args: - rel_path (str): The relative path for the properties file. - diff: The git python diff. - - Returns: - List[ItemChange]: The changes in properties. - """ - # an item was added - if diff.change_type == 'A': - changes = get_changed(rel_path, '', get_text(diff.b_blob)) - # an item was deleted - elif diff.change_type == 'D': - changes = get_changed(rel_path, get_text(diff.a_blob), '') - # an item was modified - elif diff.change_type == 'M': - changes = get_changed(rel_path, get_text( - diff.a_blob), get_text(diff.b_blob)) - else: - changes = [] - - return changes - - -def get_rel_path(diff) -> str: - """Determines the relative path based on the git python. - - Args: - diff: The git python diff. - - Returns: - str: The determined relative path. - """ - if diff.b_path is not None: - return diff.b_path - elif diff.a_path is not None: - return diff.a_path - else: - return '' - - -def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit_2_id: str): - """Determines the changes made in '.properties-MERGED' files from one commit to another commit. - - Args: - repo_path (str): The local path to the git repo. - output_path (str): The output path for the csv file. - commit_1_id (str): The initial commit for the diff. - commit_2_id (str): The latest commit for the diff. - """ - repo = Repo(repo_path) - commit_1 = repo.commit(commit_1_id) - commit_2 = repo.commit(commit_2_id) - - diffs = commit_1.diff(commit_2) - with open(output_path, 'w', newline='') as csvfile: - writer = csv.writer(csvfile) - writer.writerow(ItemChange.get_headers()) - - for diff in diffs: - rel_path = get_rel_path(diff) - if not rel_path.endswith('.properties-MERGED'): - continue - - changes = get_changed_from_diff(rel_path, diff) - - for item_change in changes: - writer.writerow(item_change.get_row()) - - -def print_help(): - """Prints a quick help message. - """ - print("diffscript.py [path to repo] [csv output path] [commit for previous release] [commit for current release (optional; defaults to 'HEAD')]") - - -def main(): - if len(sys.argv) <= 3: - print_help() - sys.exit(1) - - repo_path = sys.argv[1] - output_path = sys.argv[2] - commit_1_id = sys.argv[3] - commit_2_id = sys.argv[4] if len(sys.argv) > 4 else 'HEAD' - - write_diff_to_csv(repo_path, output_path, commit_1_id, commit_2_id) - - sys.exit(0) - - -if __name__ == "__main__": - main() diff --git a/release_scripts/localization_scripts/.gitignore b/release_scripts/localization_scripts/.gitignore new file mode 100644 index 0000000000..341e31ecbc --- /dev/null +++ b/release_scripts/localization_scripts/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +.idea \ No newline at end of file diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py new file mode 100644 index 0000000000..80fa767430 --- /dev/null +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -0,0 +1,43 @@ +"""This script finds all '.properties-MERGED' files and writes relative path, key, and value to a CSV file. +This script requires the python libraries: gitpython and jproperties. As a consequence, it also requires +git >= 1.7.0 and python >= 3.4. +""" + +import sys +from gitutil import get_property_file_entries +from csvutil import records_to_csv +import argparse + + +def write_items_to_csv(repo_path: str, output_path: str): + """Determines the contents of '.properties-MERGED' files and writes to a csv file. + + Args: + repo_path (str): The local path to the git repo. + output_path (str): The output path for the csv file. + """ + + rows = [['Relative path', 'Key', 'Value']] + for entry in get_property_file_entries(repo_path): + rows.append([entry.rel_path, entry.key, entry.value]) + + records_to_csv(output_path, rows) + + +def main(): + parser = argparse.ArgumentParser(description='Gathers all key-value pairs within .properties-MERGED files into ' + + 'one csv file.') + parser.add_argument(dest='repo_path', type=str, help='The path to the repo.') + parser.add_argument(dest='output_path', type=str, help='The path to the output csv file.') + + args = parser.parse_args() + repo_path = args.repo_path + output_path = args.output_path + + write_items_to_csv(repo_path, output_path) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/release_scripts/localization_scripts/csvutil.py b/release_scripts/localization_scripts/csvutil.py new file mode 100644 index 0000000000..5e0309998e --- /dev/null +++ b/release_scripts/localization_scripts/csvutil.py @@ -0,0 +1,47 @@ +"""Provides tools for parsing and writing to a csv file. +""" + +from typing import List, Iterable, Iterator +import csv +import os + + +def records_to_csv(output_path: str, rows: Iterable[List[str]]): + """Writes rows to a csv file at the specified path. + + Args: + output_path (str): The path where the csv file will be written. + rows (List[List[str]]): The rows to be written. Each row of a + list of strings will be written according + to their index (i.e. column 3 will be index 2). + """ + + parent_dir, file = os.path.split(output_path) + if not os.path.exists(parent_dir): + os.makedirs(parent_dir) + + with open(output_path, 'w', encoding="utf-8", newline='') as csvfile: + writer = csv.writer(csvfile) + + for row in rows: + writer.writerow(row) + + +def csv_to_records(input_path: str, header_row: bool) -> Iterator[List[str]]: + """Writes rows to a csv file at the specified path. + + Args: + input_path (str): The path where the csv file will be written. + header_row (bool): The rows to be written. Each row of a + list of strings will be written according + to their index (i.e. column 3 will be index 2). + """ + + with open(input_path, encoding='utf-8') as csv_file: + csv_reader = csv.reader(csv_file, delimiter=',') + + for row in csv_reader: + if header_row: + header_row = False + else: + yield row diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py new file mode 100644 index 0000000000..d7b679d9b3 --- /dev/null +++ b/release_scripts/localization_scripts/diffscript.py @@ -0,0 +1,61 @@ +"""This script determines the updated, added, and deleted properties from the '.properties-MERGED' files +and generates a csv file containing the items changed. This script requires the python libraries: +gitpython and jproperties. As a consequence, it also requires git >= 1.7.0 and python >= 3.4. +""" + +import sys +from gitutil import get_property_files_diff, get_commit_id +from itemchange import ItemChange +from csvutil import records_to_csv +import argparse + + +def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit_2_id: str, show_commits: bool): + """Determines the changes made in '.properties-MERGED' files from one commit to another commit. + + Args: + repo_path (str): The local path to the git repo. + output_path (str): The output path for the csv file. + commit_1_id (str): The initial commit for the diff. + commit_2_id (str): The latest commit for the diff. + show_commits (bool): show commits in the header row. + """ + + row_header = ItemChange.get_headers() + if show_commits: + row_header += [get_commit_id(repo_path, commit_1_id), get_commit_id(repo_path, commit_2_id)] + + rows = [row_header] + + rows += map(lambda item_change: item_change.get_row(), + get_property_files_diff(repo_path, commit_1_id, commit_2_id)) + + records_to_csv(output_path, rows) + + +def main(): + parser = argparse.ArgumentParser(description="determines the updated, added, and deleted properties from the " + + "'.properties-MERGED' files and generates a csv file containing " + + "the items changed.") + parser.add_argument(dest='repo_path', type=str, help='The path to the repo.') + parser.add_argument(dest='output_path', type=str, help='The path to the output csv file.') + parser.add_argument(dest='commit_1_id', type=str, help='The commit for previous release.') + parser.add_argument('-lc', '--latest_commit', dest='commit_2_id', type=str, default='HEAD', required=False, + help='The commit for current release.') + parser.add_argument('-nc', '--no_commits', dest='no_commits', action='store_true', default=False, + required=False, help="Suppresses adding commits to the generated csv header.") + + args = parser.parse_args() + repo_path = args.repo_path + output_path = args.output_path + commit_1_id = args.commit_1_id + commit_2_id = args.commit_2_id + show_commits = not args.no_commits + + write_diff_to_csv(repo_path, output_path, commit_1_id, commit_2_id, show_commits) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/release_scripts/localization_scripts/gitutil.py b/release_scripts/localization_scripts/gitutil.py new file mode 100644 index 0000000000..40b9a15adb --- /dev/null +++ b/release_scripts/localization_scripts/gitutil.py @@ -0,0 +1,147 @@ +from git import Repo, Diff, Blob +from typing import List, Union, Iterator, Tuple, Any +from itemchange import ItemChange, get_changed +from pathlib import Path +from propentry import PropEntry +from propsutil import DEFAULT_PROPS_EXTENSION, get_entry_dict + + +def get_text(blob: Blob) -> str: + return blob.data_stream.read().decode('utf-8') + + +def get_changed_from_diff(rel_path: str, diff: Diff) -> List[ItemChange]: + """Determines changes from a git python diff. + + Args: + rel_path (str): The relative path for the properties file. + diff (Diff): The git python diff. + + Returns: + List[ItemChange]: The changes in properties. + """ + + # an item was added + if diff.change_type == 'A': + changes = get_changed(rel_path, '', get_text(diff.b_blob)) + # an item was deleted + elif diff.change_type == 'D': + changes = get_changed(rel_path, get_text(diff.a_blob), '') + # an item was modified + elif diff.change_type == 'M': + changes = get_changed(rel_path, get_text( + diff.a_blob), get_text(diff.b_blob)) + else: + changes = [] + + return changes + + +def get_rel_path(diff: Diff) -> Union[str, None]: + """Determines the relative path based on the git python. + + Args: + diff: The git python diff. + + Returns: + str: The determined relative path. + """ + if diff.b_path is not None: + return diff.b_path + elif diff.a_path is not None: + return diff.a_path + else: + return None + + +def get_diff(repo_path: str, commit_1_id: str, commit_2_id: str) -> Any: + """Determines the diff between two commits. + + Args: + repo_path (str): The local path to the git repo. + commit_1_id (str): The initial commit for the diff. + commit_2_id (str): The latest commit for the diff. + + Returns: + The determined diff. + """ + repo = Repo(repo_path) + commit_1 = repo.commit(commit_1_id) + commit_2 = repo.commit(commit_2_id) + return commit_1.diff(commit_2) + + +def get_commit_id(repo_path: str, commit_id: str) -> str: + """Determines the hash for head commit. This does things like fetch the id of head if 'HEAD' is provided. + + Args: + repo_path: The path to the repo. + commit_id: The id for the commit. + + Returns: + The hash for the commit in the repo. + """ + repo = Repo(repo_path) + commit = repo.commit(commit_id) + return str(commit.hexsha) + + +def get_property_files_diff(repo_path: str, commit_1_id: str, commit_2_id: str, + property_file_extension: str = DEFAULT_PROPS_EXTENSION) -> Iterator[ItemChange]: + """Determines the item changes within property files as a diff between two commits. + + Args: + repo_path (str): The repo path. + commit_1_id (str): The first git commit. + commit_2_id (str): The second git commit. + property_file_extension (str): The extension for properties files to gather. + + Returns: + All found item changes in values of keys between the property files. + """ + + diffs = get_diff(repo_path, commit_1_id, commit_2_id) + for diff in diffs: + rel_path = get_rel_path(diff) + if rel_path is None or not rel_path.endswith('.' + property_file_extension): + continue + + yield from get_changed_from_diff(rel_path, diff) + + +def list_paths(root_tree, path: Path = Path('.')) -> Iterator[Tuple[str, Blob]]: + """ + Given the root path to serve as a prefix, walks the tree of a git commit returning all files and blobs. + Args: + root_tree: The tree of the commit to walk. + path: The path to use as a prefix. + + Returns: A tuple iterator where each tuple consists of the path as a string and a blob of the file. + + """ + for blob in root_tree.blobs: + ret_item = (str(path / blob.name), blob) + yield ret_item + for tree in root_tree.trees: + yield from list_paths(tree, path / tree.name) + + +def get_property_file_entries(repo_path: str, at_commit: str = 'HEAD', + property_file_extension: str = DEFAULT_PROPS_EXTENSION) -> Iterator[PropEntry]: + """ + Retrieves all property files entries returning as an iterator of PropEntry objects. + Args: + repo_path: The path to the git repo. + at_commit: The commit to use. + property_file_extension: The extension to use for scanning for property files. + + Returns: An iterator of PropEntry objects. + + """ + repo = Repo(repo_path) + commit = repo.commit(at_commit) + for item in list_paths(commit.tree): + path, blob = item + if path.endswith(property_file_extension): + for key, val in get_entry_dict(get_text(blob)).items(): + yield PropEntry(path, key, val) diff --git a/release_scripts/localization_scripts/itemchange.py b/release_scripts/localization_scripts/itemchange.py new file mode 100644 index 0000000000..8533be9469 --- /dev/null +++ b/release_scripts/localization_scripts/itemchange.py @@ -0,0 +1,96 @@ +from typing import Iterator, List, Union +from propsutil import get_entry_dict + + +class ItemChange: + def __init__(self, rel_path: str, key: str, prev_val: str, cur_val: str): + """Describes the change that occurred for a particular key of a properties file. + + Args: + rel_path (str): The relative path of the properties file. + key (str): The key in the properties file. + prev_val (str): The previous value for the key. + cur_val (str): The current value for the key. + """ + self.rel_path = rel_path + self.key = key + self.prev_val = prev_val + self.cur_val = cur_val + if ItemChange.has_str_content(cur_val) and not ItemChange.has_str_content(prev_val): + self.type = 'ADDITION' + elif not ItemChange.has_str_content(cur_val) and ItemChange.has_str_content(prev_val): + self.type = 'DELETION' + else: + self.type = 'CHANGE' + + @staticmethod + def has_str_content(content: str): + """Determines whether or not the content is empty or None. + + Args: + content (str): The text. + + Returns: + bool: Whether or not it has content. + """ + return content is not None and len(content.strip()) > 0 + + @staticmethod + def get_headers() -> List[str]: + """Returns the csv headers to insert when serializing a list of ItemChange objects to csv. + + Returns: + List[str]: The column headers + """ + return ['Relative Path', 'Key', 'Change Type', 'Previous Value', 'Current Value'] + + def get_row(self) -> List[str]: + """Returns the list of values to be entered as a row in csv serialization. + + Returns: + List[str]: The list of values to be entered as a row in csv serialization. + """ + return [ + self.rel_path, + self.key, + self.type, + self.prev_val, + self.cur_val] + + +def get_item_change(rel_path: str, key: str, prev_val: str, cur_val: str) -> Union[ItemChange, None]: + """Returns an ItemChange object if the previous value is not equal to the current value. + + Args: + rel_path (str): The relative path for the properties file. + key (str): The key within the properties file for this potential change. + prev_val (str): The previous value. + cur_val (str): The current value. + + Returns: + ItemChange: The ItemChange object or None if values are the same. + """ + if prev_val == cur_val: + return None + else: + return ItemChange(rel_path, key, prev_val, cur_val) + + +def get_changed(rel_path: str, a_str: str, b_str: str) -> Iterator[ItemChange]: + """Given the relative path of the properties file that + + Args: + rel_path (str): The relative path for the properties file. + a_str (str): The string representing the original state of the file. + b_str (str): The string representing the current state of the file. + + Returns: + List[ItemChange]: The changes determined. + """ + print('Retrieving changes for {}...'.format(rel_path)) + a_dict = get_entry_dict(a_str) + b_dict = get_entry_dict(b_str) + all_keys = set().union(a_dict.keys(), b_dict.keys()) + mapped = map(lambda key: get_item_change( + rel_path, key, a_dict.get(key), b_dict.get(key)), all_keys) + return filter(lambda entry: entry is not None, mapped) diff --git a/release_scripts/localization_scripts/propentry.py b/release_scripts/localization_scripts/propentry.py new file mode 100644 index 0000000000..4821f4bc5f --- /dev/null +++ b/release_scripts/localization_scripts/propentry.py @@ -0,0 +1,14 @@ +class PropEntry: + def __init__(self, rel_path: str, key: str, value: str, should_delete: bool = False): + """Defines a property file entry to be updated in a property file. + + Args: + rel_path (str): The relative path for the property file. + key (str): The key for the entry. + value (str): The value for the entry. + should_delete (bool, optional): Whether or not the key should simply be deleted. Defaults to False. + """ + self.rel_path = rel_path + self.key = key + self.value = value + self.should_delete = should_delete diff --git a/release_scripts/localization_scripts/propsutil.py b/release_scripts/localization_scripts/propsutil.py new file mode 100644 index 0000000000..b0b301c714 --- /dev/null +++ b/release_scripts/localization_scripts/propsutil.py @@ -0,0 +1,65 @@ +"""Provides tools for reading from and writing to java properties files. +""" + +from typing import Dict, Union, IO +from jproperties import Properties +import os +from os import path + +# The default extension for property files in autopsy repo +DEFAULT_PROPS_EXTENSION = 'properties-MERGED' + + +def get_entry_dict(file_contents: Union[str, IO]) -> Dict[str, str]: + """Retrieves a dictionary mapping the properties represented in the string. + + Args: + file_contents: The string of the properties file or the file handle. + + Returns: + Dict[str,str]: The mapping of keys to values in that properties file. + """ + + props = Properties() + props.load(file_contents, "utf-8") + return props.properties + + +def set_entry_dict(contents: Dict[str, str], file_path: str): + """Sets the property file to the key-value pairs of the contents dictionary. + + Args: + contents (Dict[str, str]): The dictionary whose contents will be the key value pairs of the properties file. + file_path (str): The path to the properties file to create. + """ + + props = Properties() + for key, val in contents.items(): + props[key] = val + + parent_dir, file = os.path.split(file_path) + if not os.path.exists(parent_dir): + os.makedirs(parent_dir) + + with open(file_path, "wb") as f: + props.store(f) + + +def update_entry_dict(contents: Dict[str, str], file_path: str): + """Updates the properties file at the given location with the key-value properties of contents. + Creates a new properties file at given path if none exists. + + Args: + contents (Dict[str, str]): The dictionary whose contents will be the key value pairs of the properties file. + file_path (str): The path to the properties file to create. + """ + contents_to_edit = contents.copy() + + if path.isfile(file_path): + cur_dict = get_entry_dict(file_path) + for cur_key, cur_val in cur_dict.values(): + # only update contents if contents does not already have key + if cur_key not in contents_to_edit: + contents_to_edit[cur_key] = cur_val + + set_entry_dict(contents_to_edit, file_path) diff --git a/release_scripts/localization_scripts/updatepropsscript.py b/release_scripts/localization_scripts/updatepropsscript.py new file mode 100644 index 0000000000..f575ee847d --- /dev/null +++ b/release_scripts/localization_scripts/updatepropsscript.py @@ -0,0 +1,240 @@ +"""This script finds all '.properties-MERGED' files and writes relative path, key, and value to a CSV file. +This script requires the python libraries: jproperties. It also requires Python 3.x. +""" + +from typing import List, Dict, Tuple, Callable, Iterator +import sys +import os +from propsutil import set_entry_dict, get_entry_dict +from csvutil import csv_to_records +from propentry import PropEntry +import argparse + + +def write_prop_entries(entries: Iterator[PropEntry], repo_path: str): + """Writes property entry items to their expected relative path within the repo path. + Previously existing files will be overwritten and prop entries marked as should_be_deleted will + not be included. + + Args: + entries (List[PropEntry]): the prop entry items to write to disk. + repo_path (str): The path to the git repo. + """ + items_by_file = get_by_file(entries) + for rel_path, (entries, ignored) in items_by_file.items(): + abs_path = os.path.join(repo_path, rel_path) + set_entry_dict(entries, abs_path) + + +def update_prop_entries(entries: Iterator[PropEntry], repo_path: str): + """Updates property entry items to their expected relative path within the repo path. The union of + entries provided and any previously existing entries will be created. Keys marked for deletion will be + removed from the generated property files. + + Args: + entries (List[PropEntry]): the prop entry items to write to disk. + repo_path (str): The path to the git repo. + """ + items_by_file = get_by_file(entries) + for rel_path, (entries, to_delete) in items_by_file.items(): + abs_path = os.path.join(repo_path, rel_path) + + if os.path.isfile(abs_path): + with open(abs_path, "rb") as f: + prop_items = get_entry_dict(f) + else: + prop_items = {} + + for key_to_delete in to_delete: + if key_to_delete in prop_items: + del prop_items[key_to_delete] + + for key, val in entries.items(): + prop_items[key] = val + + set_entry_dict(prop_items, abs_path) + + +def get_by_file(entries: Iterator[PropEntry]) -> Dict[str, Tuple[Dict[str, str], List[str]]]: + """Sorts a prop entry list by file. The return type is a dictionary mapping + the file path to a tuple containing the key-value pairs to be updated and a + list of keys to be deleted. + + Args: + entries (List[PropEntry]): The entries to be sorted. + + Returns: + Dict[str, Tuple[Dict[str,str], List[str]]]: A dictionary mapping + the file path to a tuple containing the key-value pairs to be updated and a + list of keys to be deleted. + """ + to_ret = {} + for prop_entry in entries: + rel_path = prop_entry.rel_path + key = prop_entry.key + value = prop_entry.value + + if rel_path not in to_ret: + to_ret[rel_path] = ({}, []) + + if prop_entry.should_delete: + to_ret[rel_path][1].append(prop_entry.key) + else: + to_ret[rel_path][0][key] = value + + return to_ret + + +def idx_bounded(num: int, max_exclusive: int) -> bool: + return 0 <= num < max_exclusive + + +def get_prop_entry(row: List[str], + path_idx: int = 0, + key_idx: int = 1, + value_idx: int = 2, + should_delete_converter: Callable[[List[str]], bool] = None, + path_converter: Callable[[str], str] = None) -> PropEntry: + """Parses a PropEntry object from a row of values in a csv. + + Args: + row (List[str]): The csv file row to parse. + path_idx (int, optional): The column index for the relative path of the properties file. Defaults to 0. + key_idx (int, optional): The column index for the properties key. Defaults to 1. + value_idx (int, optional): The column index for the properties value. Defaults to 2. + should_delete_converter (Callable[[List[str]], bool], optional): If not None, this determines if the key should + be deleted from the row values. Defaults to None. + path_converter (Callable[[str], str], optional): If not None, this determines the relative path to use in the + created PropEntry given the original relative path. Defaults to None. + + Returns: + PropEntry: The generated prop entry object. + """ + + path = row[path_idx] if idx_bounded(path_idx, len(row)) else None + if path_converter is not None: + path = path_converter(path) + + key = row[key_idx] if idx_bounded(key_idx, len(row)) else None + value = row[value_idx] if idx_bounded(value_idx, len(row)) else None + should_delete = False if should_delete_converter is None else should_delete_converter(row) + return PropEntry(path, key, value, should_delete) + + +def get_prop_entries(rows: List[List[str]], + path_idx: int = 0, + key_idx: int = 1, + value_idx: int = 2, + should_delete_converter: Callable[[List[str]], bool] = None, + path_converter: Callable[[str], str] = None) -> Iterator[PropEntry]: + + """Parses PropEntry objects from rows of values in a csv. + + Args: + rows (List[List[str]]): The csv file rows to parse. + path_idx (int, optional): The column index for the relative path of the properties file. Defaults to 0. + key_idx (int, optional): The column index for the properties key. Defaults to 1. + value_idx (int, optional): The column index for the properties value. Defaults to 2. + should_delete_converter (Callable[[List[str]], bool], optional): If not None, this determines if the key should + be deleted from the row values. Defaults to None. + path_converter (Callable[[str], str], optional): If not None, this determines the relative path to use in the + created PropEntry given the original relative path. Defaults to None. + + Returns: + List[PropEntry]: The generated prop entry objects. + """ + return map(lambda row: get_prop_entry( + row, path_idx, key_idx, value_idx, should_delete_converter, path_converter), + rows) + + +def get_should_deleted(row_items: List[str], requested_idx: int) -> bool: + """If there is a value at row_items[requested_idx] and that value is not empty, then this will return true. + + Args: + row_items (List[str]): The row items. + requested_idx (int): The index specifying if the property should be deleted. + + Returns: + bool: True if the row specifies it should be deleted. + """ + if idx_bounded(requested_idx, len(row_items)) and len((row_items[requested_idx].strip())) > 0: + return True + else: + return False + + +def get_new_rel_path(orig_path: str, new_filename: str) -> str: + """Obtains a new relative path. This tries to determine if the provided path is a directory or filename (has an + extension containing '.') then constructs the new path with the old parent directory and the new filename. + + Args: + orig_path (str): The original path. + new_filename (str): The new filename to use. + + Returns: + str: The new path. + """ + potential_parent_dir, orig_file = os.path.split(orig_path) + parent_dir = orig_path if '.' not in orig_file else potential_parent_dir + return os.path.join(parent_dir, new_filename) + + +def main(): + parser = argparse.ArgumentParser(description='Updates properties files in the autopsy git repo.') + + parser.add_argument(dest='csv_file', type=str, help='The path to the csv file.') + parser.add_argument(dest='repo_path', type=str, help='The path to the repo.') + parser.add_argument('-p', '--path_idx', dest='path_idx', action='store', type=int, default=0, required=False, + help='The column index in the csv file providing the relative path to the properties file.') + parser.add_argument('-k', '--key_idx', dest='key_idx', action='store', type=int, default=1, required=False, + help='The column index in the csv file providing the key within the properties file.') + parser.add_argument('-v', '--value_idx', dest='value_idx', action='store', type=int, default=2, required=False, + help='The column index in the csv file providing the value within the properties file.') + parser.add_argument('-d', '--should_delete_idx', dest='should_delete_idx', action='store', type=int, default=None, + required=False, help='The column index in the csv file providing whether or not the file ' + + 'should be deleted. Any non-blank content will be treated as True.') + parser.add_argument('-f', '--file_rename', dest='file_rename', action='store', type=str, default=None, + required=False, help='If specified, the properties file will be renamed to the argument' + + ' preserving the specified relative path.') + parser.add_argument('--has_no_header', dest='has_no_header', action='store_true', default=False, required=False, + help='Specify whether or not there is a header within the csv file.') + parser.add_argument('-o', '--should_overwrite', dest='should_overwrite', action='store_true', default=False, + required=False, help="Whether or not to overwrite the previously existing properties files" + + " ignoring previously existing values.") + + args = parser.parse_args() + + repo_path = args.repo_path + input_path = args.csv_file + path_idx = args.path_idx + key_idx = args.key_idx + value_idx = args.value_idx + has_header = not args.has_no_header + overwrite = args.should_overwrite + + if args.should_delete_idx is None: + should_delete_converter = None + else: + def should_delete_converter(row_items: List[str]): + return get_should_deleted(row_items, args.should_delete_idx) + + if args.file_rename is None: + path_converter = None + else: + def path_converter(orig_path: str): + return get_new_rel_path(orig_path, args.file_rename) + + all_items = list(csv_to_records(input_path, has_header)) + prop_entries = get_prop_entries(all_items, path_idx, key_idx, value_idx, should_delete_converter, path_converter) + + if overwrite: + write_prop_entries(prop_entries, repo_path) + else: + update_prop_entries(prop_entries, repo_path) + + sys.exit(0) + + +if __name__ == "__main__": + main() From bb8d370a3f803e8261d5caa4422ffed4482f8830 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 10 Jul 2020 14:31:18 -0400 Subject: [PATCH 02/42] commits in headers --- .../localization_scripts/allbundlesscript.py | 19 ++++++++++++++----- .../localization_scripts/gitutil.py | 3 +++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py index 80fa767430..c4273a7a5d 100644 --- a/release_scripts/localization_scripts/allbundlesscript.py +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -1,23 +1,29 @@ """This script finds all '.properties-MERGED' files and writes relative path, key, and value to a CSV file. This script requires the python libraries: gitpython and jproperties. As a consequence, it also requires -git >= 1.7.0 and python >= 3.4. +git >= 1.7.0 and python >= 3.4. This script relies on fetching 'HEAD' from current branch. So make sure +repo is on correct branch (i.e. develop). """ import sys -from gitutil import get_property_file_entries +from gitutil import get_property_file_entries, get_commit_id from csvutil import records_to_csv import argparse -def write_items_to_csv(repo_path: str, output_path: str): +def write_items_to_csv(repo_path: str, output_path: str, show_commit: bool): """Determines the contents of '.properties-MERGED' files and writes to a csv file. Args: repo_path (str): The local path to the git repo. output_path (str): The output path for the csv file. + show_commit (bool): Whether or not to include the commit id in the header """ - rows = [['Relative path', 'Key', 'Value']] + row_header = ['Relative path', 'Key', 'Value'] + if show_commit: + row_header.append(get_commit_id(repo_path, 'HEAD')) + + rows = [row_header] for entry in get_property_file_entries(repo_path): rows.append([entry.rel_path, entry.key, entry.value]) @@ -29,12 +35,15 @@ def main(): 'one csv file.') parser.add_argument(dest='repo_path', type=str, help='The path to the repo.') parser.add_argument(dest='output_path', type=str, help='The path to the output csv file.') + parser.add_argument('-nc', '--no_commit', dest='no_commit', action='store_true', default=False, + required=False, help="Suppresses adding commits to the generated csv header.") args = parser.parse_args() repo_path = args.repo_path output_path = args.output_path + show_commit = not args.no_commit - write_items_to_csv(repo_path, output_path) + write_items_to_csv(repo_path, output_path, show_commit) sys.exit(0) diff --git a/release_scripts/localization_scripts/gitutil.py b/release_scripts/localization_scripts/gitutil.py index 40b9a15adb..dd7b13632f 100644 --- a/release_scripts/localization_scripts/gitutil.py +++ b/release_scripts/localization_scripts/gitutil.py @@ -112,6 +112,8 @@ def get_property_files_diff(repo_path: str, commit_1_id: str, commit_2_id: str, def list_paths(root_tree, path: Path = Path('.')) -> Iterator[Tuple[str, Blob]]: """ Given the root path to serve as a prefix, walks the tree of a git commit returning all files and blobs. + Repurposed from: https://www.enricozini.org/blog/2019/debian/gitpython-list-all-files-in-a-git-commit/ + Args: root_tree: The tree of the commit to walk. path: The path to use as a prefix. @@ -130,6 +132,7 @@ def get_property_file_entries(repo_path: str, at_commit: str = 'HEAD', property_file_extension: str = DEFAULT_PROPS_EXTENSION) -> Iterator[PropEntry]: """ Retrieves all property files entries returning as an iterator of PropEntry objects. + Args: repo_path: The path to the git repo. at_commit: The commit to use. From 54cfd1cd51dbdca706647890342615a277539254 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 13 Jul 2020 10:15:29 -0400 Subject: [PATCH 03/42] use parent git repos --- release_scripts/localization_scripts/gitutil.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release_scripts/localization_scripts/gitutil.py b/release_scripts/localization_scripts/gitutil.py index dd7b13632f..65db268fc8 100644 --- a/release_scripts/localization_scripts/gitutil.py +++ b/release_scripts/localization_scripts/gitutil.py @@ -65,7 +65,7 @@ def get_diff(repo_path: str, commit_1_id: str, commit_2_id: str) -> Any: Returns: The determined diff. """ - repo = Repo(repo_path) + repo = Repo(repo_path, search_parent_directories=True) commit_1 = repo.commit(commit_1_id) commit_2 = repo.commit(commit_2_id) return commit_1.diff(commit_2) @@ -81,7 +81,7 @@ def get_commit_id(repo_path: str, commit_id: str) -> str: Returns: The hash for the commit in the repo. """ - repo = Repo(repo_path) + repo = Repo(repo_path, search_parent_directories=True) commit = repo.commit(commit_id) return str(commit.hexsha) @@ -141,7 +141,7 @@ def get_property_file_entries(repo_path: str, at_commit: str = 'HEAD', Returns: An iterator of PropEntry objects. """ - repo = Repo(repo_path) + repo = Repo(repo_path, search_parent_directories=True) commit = repo.commit(at_commit) for item in list_paths(commit.tree): path, blob = item From f24a5a30e2d7b10516ab342317473913c55a2780 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 13 Jul 2020 11:39:18 -0400 Subject: [PATCH 04/42] working through commit args --- .../localization_scripts/diffscript.py | 29 +++++++++++++++---- .../localization_scripts/updatepropsscript.py | 5 ++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index d7b679d9b3..efbb4b0c76 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -9,6 +9,8 @@ from itemchange import ItemChange from csvutil import records_to_csv import argparse +from langpropsutil import get_commit_for_language, LANG_FILENAME + def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit_2_id: str, show_commits: bool): """Determines the changes made in '.properties-MERGED' files from one commit to another commit. @@ -36,19 +38,36 @@ def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit def main(): parser = argparse.ArgumentParser(description="determines the updated, added, and deleted properties from the " + "'.properties-MERGED' files and generates a csv file containing " + - "the items changed.") - parser.add_argument(dest='repo_path', type=str, help='The path to the repo.') + "the items changed.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument(dest='output_path', type=str, help='The path to the output csv file.') - parser.add_argument(dest='commit_1_id', type=str, help='The commit for previous release.') - parser.add_argument('-lc', '--latest_commit', dest='commit_2_id', type=str, default='HEAD', required=False, + + parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, + help='The path to the repo. If not specified, path of script is used.') + parser.add_argument('-fc', '--first-commit', dest='commit_1_id', type=str, required=False, + help='The commit for previous release. This flag or the language flag need to be specified' + + ' in order to determine a start point for the difference.') + parser.add_argument('-lc', '--latest-commit', dest='commit_2_id', type=str, default='HEAD', required=False, help='The commit for current release.') - parser.add_argument('-nc', '--no_commits', dest='no_commits', action='store_true', default=False, + parser.add_argument('-nc', '--no-commits', dest='no_commits', action='store_true', default=False, required=False, help="Suppresses adding commits to the generated csv header.") + parser.add_argument('-l', '--language', dest='language', type=str, default='HEAD', required=False, + help='Specify the language in order to determine the first commit to use (i.e. \'ja\' for ' + + 'Japanese. This flag overrides the first-commit flag.') args = parser.parse_args() repo_path = args.repo_path output_path = args.output_path commit_1_id = args.commit_1_id + if args.language is not None: + commit_1_id = get_commit_for_language(args.language) + + if commit_1_id is None: + print('Either the first commit or language flag need to be specified. If specified, the language file, ' + + LANG_FILENAME + ', may not have the latest commit for the language.', file=sys.stderr) + parser.print_help(sys.stderr) + sys.exit(1) + commit_2_id = args.commit_2_id show_commits = not args.no_commits diff --git a/release_scripts/localization_scripts/updatepropsscript.py b/release_scripts/localization_scripts/updatepropsscript.py index f575ee847d..2b70fa00dd 100644 --- a/release_scripts/localization_scripts/updatepropsscript.py +++ b/release_scripts/localization_scripts/updatepropsscript.py @@ -194,6 +194,10 @@ def main(): parser.add_argument('-d', '--should_delete_idx', dest='should_delete_idx', action='store', type=int, default=None, required=False, help='The column index in the csv file providing whether or not the file ' + 'should be deleted. Any non-blank content will be treated as True.') + parser.add_argument('-c', '--commit_idx', dest='latest_commit_idx', action='store', type=int, default=3, + required=False, help='The column index in the csv file providing the commit for which this ' + + 'update applies. The commit should be located in the header row' + ) parser.add_argument('-f', '--file_rename', dest='file_rename', action='store', type=str, default=None, required=False, help='If specified, the properties file will be renamed to the argument' + ' preserving the specified relative path.') @@ -203,6 +207,7 @@ def main(): required=False, help="Whether or not to overwrite the previously existing properties files" + " ignoring previously existing values.") + args = parser.parse_args() repo_path = args.repo_path From 32b96584ce0efbccb2b801eb41cdb5c8450a07f3 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 13 Jul 2020 11:39:30 -0400 Subject: [PATCH 05/42] working through commit args --- release_scripts/localization_scripts/langpropsutil.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 release_scripts/localization_scripts/langpropsutil.py diff --git a/release_scripts/localization_scripts/langpropsutil.py b/release_scripts/localization_scripts/langpropsutil.py new file mode 100644 index 0000000000..3124137b9c --- /dev/null +++ b/release_scripts/localization_scripts/langpropsutil.py @@ -0,0 +1,7 @@ +LANG_FILENAME = 'lastupdated.properties' + +def get_commit_for_language(language: str) -> str: + pass + +def set_commit_for_language(language: str, latest_commit: str): + pass \ No newline at end of file From 8f9d24958575aa7bb99650755655f779162e11ea Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 13 Jul 2020 15:27:16 -0400 Subject: [PATCH 06/42] updates --- release_scripts/localization_scripts/updatepropsscript.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/release_scripts/localization_scripts/updatepropsscript.py b/release_scripts/localization_scripts/updatepropsscript.py index 2b70fa00dd..fec721d9f7 100644 --- a/release_scripts/localization_scripts/updatepropsscript.py +++ b/release_scripts/localization_scripts/updatepropsscript.py @@ -196,8 +196,7 @@ def main(): 'should be deleted. Any non-blank content will be treated as True.') parser.add_argument('-c', '--commit_idx', dest='latest_commit_idx', action='store', type=int, default=3, required=False, help='The column index in the csv file providing the commit for which this ' + - 'update applies. The commit should be located in the header row' - ) + 'update applies. The commit should be located in the header row. ') parser.add_argument('-f', '--file_rename', dest='file_rename', action='store', type=str, default=None, required=False, help='If specified, the properties file will be renamed to the argument' + ' preserving the specified relative path.') From f8cff3ff1defb8bd13e7462e313507349cfa18b1 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 13 Jul 2020 16:33:11 -0400 Subject: [PATCH 07/42] working through last updated items --- .../localization_scripts/allbundlesscript.py | 6 ++- .../localization_scripts/csvutil.py | 15 +++--- .../localization_scripts/diffscript.py | 3 +- .../localization_scripts/langpropsutil.py | 26 ++++++++-- .../localization_scripts/propsutil.py | 12 +++++ .../localization_scripts/updatepropsscript.py | 52 +++++++++++++------ 6 files changed, 85 insertions(+), 29 deletions(-) diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py index c4273a7a5d..2c73565939 100644 --- a/release_scripts/localization_scripts/allbundlesscript.py +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -7,6 +7,7 @@ repo is on correct branch (i.e. develop). import sys from gitutil import get_property_file_entries, get_commit_id from csvutil import records_to_csv +import pathlib import argparse @@ -33,13 +34,14 @@ def write_items_to_csv(repo_path: str, output_path: str, show_commit: bool): def main(): parser = argparse.ArgumentParser(description='Gathers all key-value pairs within .properties-MERGED files into ' + 'one csv file.') - parser.add_argument(dest='repo_path', type=str, help='The path to the repo.') parser.add_argument(dest='output_path', type=str, help='The path to the output csv file.') + parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, + help='The path to the repo. If not specified, path of script is used.') parser.add_argument('-nc', '--no_commit', dest='no_commit', action='store_true', default=False, required=False, help="Suppresses adding commits to the generated csv header.") args = parser.parse_args() - repo_path = args.repo_path + repo_path = args.repo_path if args.repo_path is not None else str(pathlib.Path(__file__).parent.absolute()) output_path = args.output_path show_commit = not args.no_commit diff --git a/release_scripts/localization_scripts/csvutil.py b/release_scripts/localization_scripts/csvutil.py index 5e0309998e..4349e781a6 100644 --- a/release_scripts/localization_scripts/csvutil.py +++ b/release_scripts/localization_scripts/csvutil.py @@ -1,7 +1,7 @@ """Provides tools for parsing and writing to a csv file. """ -from typing import List, Iterable, Iterator +from typing import List, Iterable, Tuple import csv import os @@ -27,21 +27,24 @@ def records_to_csv(output_path: str, rows: Iterable[List[str]]): writer.writerow(row) -def csv_to_records(input_path: str, header_row: bool) -> Iterator[List[str]]: +def csv_to_records(input_path: str, header_row: bool) -> Tuple[List[List[str]], List[str]]: """Writes rows to a csv file at the specified path. Args: input_path (str): The path where the csv file will be written. - header_row (bool): The rows to be written. Each row of a - list of strings will be written according - to their index (i.e. column 3 will be index 2). + header_row (bool): Whether or not there is a header row to be skipped. """ with open(input_path, encoding='utf-8') as csv_file: csv_reader = csv.reader(csv_file, delimiter=',') + header = None + results = [] for row in csv_reader: if header_row: + header = row header_row = False else: - yield row + results.append(row) + + return results, header diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index efbb4b0c76..3351c75abb 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -8,6 +8,7 @@ from gitutil import get_property_files_diff, get_commit_id from itemchange import ItemChange from csvutil import records_to_csv import argparse +import pathlib from langpropsutil import get_commit_for_language, LANG_FILENAME @@ -56,7 +57,7 @@ def main(): 'Japanese. This flag overrides the first-commit flag.') args = parser.parse_args() - repo_path = args.repo_path + repo_path = args.repo_path if args.repo_path is not None else str(pathlib.Path(__file__).parent.absolute()) output_path = args.output_path commit_1_id = args.commit_1_id if args.language is not None: diff --git a/release_scripts/localization_scripts/langpropsutil.py b/release_scripts/localization_scripts/langpropsutil.py index 3124137b9c..43e8707d1c 100644 --- a/release_scripts/localization_scripts/langpropsutil.py +++ b/release_scripts/localization_scripts/langpropsutil.py @@ -1,7 +1,27 @@ +from typing import Union +from propsutil import get_entry_dict +import pathlib +from os import path + + LANG_FILENAME = 'lastupdated.properties' -def get_commit_for_language(language: str) -> str: - pass +def get_last_update_key(language: str) -> str: + return "bundles.{lang}.lastupdated".format({lang=language}) + +def get_commit_for_language(language: str) -> Union[str, None]: + this_path = path.join(get_props_file_path(), LANG_FILENAME) + + if path.isfile(this_path): + lang_dict = get_entry_dict(this_path) + key = get_last_update_key(language) + if key in lang_dict: + return lang_dict[key] + + return None def set_commit_for_language(language: str, latest_commit: str): - pass \ No newline at end of file + pass + +def get_props_file_path() -> str: + return str(pathlib.Path(__file__).parent.absolute()) diff --git a/release_scripts/localization_scripts/propsutil.py b/release_scripts/localization_scripts/propsutil.py index b0b301c714..af48e8173c 100644 --- a/release_scripts/localization_scripts/propsutil.py +++ b/release_scripts/localization_scripts/propsutil.py @@ -10,6 +10,18 @@ from os import path DEFAULT_PROPS_EXTENSION = 'properties-MERGED' +def get_lang_bundle_name(language: str) -> str: + """ + Returns the bundle name for the specific language identifier provided. + Args: + language: The language identifier (i.e. 'ja' for Japanese) + + Returns: + The bundle name + """ + return 'Bundle_{lang}.properties'.format(lang=language) + + def get_entry_dict(file_contents: Union[str, IO]) -> Dict[str, str]: """Retrieves a dictionary mapping the properties represented in the string. diff --git a/release_scripts/localization_scripts/updatepropsscript.py b/release_scripts/localization_scripts/updatepropsscript.py index fec721d9f7..b975a5b115 100644 --- a/release_scripts/localization_scripts/updatepropsscript.py +++ b/release_scripts/localization_scripts/updatepropsscript.py @@ -5,10 +5,13 @@ This script requires the python libraries: jproperties. It also requires Python from typing import List, Dict, Tuple, Callable, Iterator import sys import os -from propsutil import set_entry_dict, get_entry_dict + +from langpropsutil import set_commit_for_language +from propsutil import set_entry_dict, get_entry_dict, get_lang_bundle_name from csvutil import csv_to_records from propentry import PropEntry import argparse +import pathlib def write_prop_entries(entries: Iterator[PropEntry], repo_path: str): @@ -184,32 +187,36 @@ def main(): parser = argparse.ArgumentParser(description='Updates properties files in the autopsy git repo.') parser.add_argument(dest='csv_file', type=str, help='The path to the csv file.') - parser.add_argument(dest='repo_path', type=str, help='The path to the repo.') - parser.add_argument('-p', '--path_idx', dest='path_idx', action='store', type=int, default=0, required=False, + + parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, + help='The path to the repo. If not specified, path of script is used.') + parser.add_argument('-p', '--path-idx', dest='path_idx', action='store', type=int, default=0, required=False, help='The column index in the csv file providing the relative path to the properties file.') - parser.add_argument('-k', '--key_idx', dest='key_idx', action='store', type=int, default=1, required=False, + parser.add_argument('-k', '--key-idx', dest='key_idx', action='store', type=int, default=1, required=False, help='The column index in the csv file providing the key within the properties file.') - parser.add_argument('-v', '--value_idx', dest='value_idx', action='store', type=int, default=2, required=False, + parser.add_argument('-v', '--value-idx', dest='value_idx', action='store', type=int, default=2, required=False, help='The column index in the csv file providing the value within the properties file.') - parser.add_argument('-d', '--should_delete_idx', dest='should_delete_idx', action='store', type=int, default=None, + parser.add_argument('-d', '--should-delete-idx', dest='should_delete_idx', action='store', type=int, default=None, required=False, help='The column index in the csv file providing whether or not the file ' + 'should be deleted. Any non-blank content will be treated as True.') - parser.add_argument('-c', '--commit_idx', dest='latest_commit_idx', action='store', type=int, default=3, + parser.add_argument('-c', '--commit-idx', dest='latest_commit_idx', action='store', type=int, default=3, required=False, help='The column index in the csv file providing the commit for which this ' + 'update applies. The commit should be located in the header row. ') - parser.add_argument('-f', '--file_rename', dest='file_rename', action='store', type=str, default=None, + parser.add_argument('-f', '--file-rename', dest='file_rename', action='store', type=str, default=None, required=False, help='If specified, the properties file will be renamed to the argument' + ' preserving the specified relative path.') - parser.add_argument('--has_no_header', dest='has_no_header', action='store_true', default=False, required=False, - help='Specify whether or not there is a header within the csv file.') - parser.add_argument('-o', '--should_overwrite', dest='should_overwrite', action='store_true', default=False, + parser.add_argument('-z', '--has-no-header', dest='has_no_header', action='store_true', default=False, + required=False, help='Specify whether or not there is a header within the csv file.') + parser.add_argument('-o', '--should-overwrite', dest='should_overwrite', action='store_true', default=False, required=False, help="Whether or not to overwrite the previously existing properties files" + " ignoring previously existing values.") - + parser.add_argument('-l', '--language', dest='language', type=str, default='HEAD', required=False, + help='Specify the language in order to update the last updated properties file and rename ' + + 'files within directories. This flag overrides the file-rename flag.') args = parser.parse_args() - repo_path = args.repo_path + repo_path = args.repo_path if args.repo_path is not None else str(pathlib.Path(__file__).parent.absolute()) input_path = args.csv_file path_idx = args.path_idx key_idx = args.key_idx @@ -217,26 +224,37 @@ def main(): has_header = not args.has_no_header overwrite = args.should_overwrite + # means of determining if a key should be deleted from a file if args.should_delete_idx is None: should_delete_converter = None else: def should_delete_converter(row_items: List[str]): return get_should_deleted(row_items, args.should_delete_idx) - if args.file_rename is None: - path_converter = None - else: + # provides the means of renaming the bundle file + if args.language is not None: + def path_converter(orig_path: str): + return get_new_rel_path(orig_path, get_lang_bundle_name(args.language)) + elif args.file_rename is not None: def path_converter(orig_path: str): return get_new_rel_path(orig_path, args.file_rename) + else: + path_converter = None - all_items = list(csv_to_records(input_path, has_header)) + # retrieve records from csv + all_items, header = list(csv_to_records(input_path, has_header)) prop_entries = get_prop_entries(all_items, path_idx, key_idx, value_idx, should_delete_converter, path_converter) + # write to files if overwrite: write_prop_entries(prop_entries, repo_path) else: update_prop_entries(prop_entries, repo_path) + # update the language last update if applicable + if args.language is not None and header is not None and len(header) > args.latest_commit_idx >= 0: + set_commit_for_language(args.language, header[args.latest_commit_idx]) + sys.exit(0) From f836176009e575824644a16b0b85afc16fdd98b3 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 14 Jul 2020 08:03:41 -0400 Subject: [PATCH 08/42] updates to include updated --- .../localization_scripts/allbundlesscript.py | 11 ++++-- .../localization_scripts/diffscript.py | 12 +++--- .../localization_scripts/envutil.py | 17 +++++++++ .../localization_scripts/gitutil.py | 3 ++ .../localization_scripts/langpropsutil.py | 38 +++++++++++-------- .../lastupdated.properties | 0 .../localization_scripts/propsutil.py | 17 +++++++++ .../localization_scripts/updatepropsscript.py | 29 +++++++------- 8 files changed, 90 insertions(+), 37 deletions(-) create mode 100644 release_scripts/localization_scripts/envutil.py create mode 100644 release_scripts/localization_scripts/lastupdated.properties diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py index 2c73565939..055cc835e8 100644 --- a/release_scripts/localization_scripts/allbundlesscript.py +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -5,9 +5,10 @@ repo is on correct branch (i.e. develop). """ import sys + +from envutil import get_proj_dir from gitutil import get_property_file_entries, get_commit_id from csvutil import records_to_csv -import pathlib import argparse @@ -25,6 +26,7 @@ def write_items_to_csv(repo_path: str, output_path: str, show_commit: bool): row_header.append(get_commit_id(repo_path, 'HEAD')) rows = [row_header] + for entry in get_property_file_entries(repo_path): rows.append([entry.rel_path, entry.key, entry.value]) @@ -32,8 +34,9 @@ def write_items_to_csv(repo_path: str, output_path: str, show_commit: bool): def main(): - parser = argparse.ArgumentParser(description='Gathers all key-value pairs within .properties-MERGED files into ' + - 'one csv file.') + parser = argparse.ArgumentParser(description='Gathers all key-value pairs within .properties-MERGED files into ' + 'one csv file.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument(dest='output_path', type=str, help='The path to the output csv file.') parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, help='The path to the repo. If not specified, path of script is used.') @@ -41,7 +44,7 @@ def main(): required=False, help="Suppresses adding commits to the generated csv header.") args = parser.parse_args() - repo_path = args.repo_path if args.repo_path is not None else str(pathlib.Path(__file__).parent.absolute()) + repo_path = args.repo_path if args.repo_path is not None else get_proj_dir() output_path = args.output_path show_commit = not args.no_commit diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index 3351c75abb..ebf50dc0c9 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -4,6 +4,8 @@ gitpython and jproperties. As a consequence, it also requires git >= 1.7.0 and """ import sys + +from envutil import get_proj_dir from gitutil import get_property_files_diff, get_commit_id from itemchange import ItemChange from csvutil import records_to_csv @@ -37,8 +39,8 @@ def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit def main(): - parser = argparse.ArgumentParser(description="determines the updated, added, and deleted properties from the " + - "'.properties-MERGED' files and generates a csv file containing " + + parser = argparse.ArgumentParser(description="determines the updated, added, and deleted properties from the " + "'.properties-MERGED' files and generates a csv file containing " "the items changed.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument(dest='output_path', type=str, help='The path to the output csv file.') @@ -46,18 +48,18 @@ def main(): parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, help='The path to the repo. If not specified, path of script is used.') parser.add_argument('-fc', '--first-commit', dest='commit_1_id', type=str, required=False, - help='The commit for previous release. This flag or the language flag need to be specified' + + help='The commit for previous release. This flag or the language flag need to be specified' ' in order to determine a start point for the difference.') parser.add_argument('-lc', '--latest-commit', dest='commit_2_id', type=str, default='HEAD', required=False, help='The commit for current release.') parser.add_argument('-nc', '--no-commits', dest='no_commits', action='store_true', default=False, required=False, help="Suppresses adding commits to the generated csv header.") parser.add_argument('-l', '--language', dest='language', type=str, default='HEAD', required=False, - help='Specify the language in order to determine the first commit to use (i.e. \'ja\' for ' + + help='Specify the language in order to determine the first commit to use (i.e. \'ja\' for ' 'Japanese. This flag overrides the first-commit flag.') args = parser.parse_args() - repo_path = args.repo_path if args.repo_path is not None else str(pathlib.Path(__file__).parent.absolute()) + repo_path = args.repo_path if args.repo_path is not None else get_proj_dir() output_path = args.output_path commit_1_id = args.commit_1_id if args.language is not None: diff --git a/release_scripts/localization_scripts/envutil.py b/release_scripts/localization_scripts/envutil.py new file mode 100644 index 0000000000..cec2a00eda --- /dev/null +++ b/release_scripts/localization_scripts/envutil.py @@ -0,0 +1,17 @@ +"""Functions relating to the project environment. +""" + +import pathlib +from typing import Union + + +def get_proj_dir(path: Union[pathlib.PurePath, str] = __file__) -> str: + """ + Gets parent directory of this file (and subsequently, the project). + Args: + path: Can be overridden to provide a different file. This will return the parent of that file in that instance. + + Returns: + The project folder or the parent folder of the file provided. + """ + return str(pathlib.Path(path).parent.absolute()) diff --git a/release_scripts/localization_scripts/gitutil.py b/release_scripts/localization_scripts/gitutil.py index 65db268fc8..9b9667d421 100644 --- a/release_scripts/localization_scripts/gitutil.py +++ b/release_scripts/localization_scripts/gitutil.py @@ -1,3 +1,6 @@ +"""Functions relating to using git and GitPython with an existing repo. +""" + from git import Repo, Diff, Blob from typing import List, Union, Iterator, Tuple, Any from itemchange import ItemChange, get_changed diff --git a/release_scripts/localization_scripts/langpropsutil.py b/release_scripts/localization_scripts/langpropsutil.py index 43e8707d1c..1104e14665 100644 --- a/release_scripts/localization_scripts/langpropsutil.py +++ b/release_scripts/localization_scripts/langpropsutil.py @@ -1,27 +1,35 @@ +"""Functions handling retrieving and storing when a language was last updated. +""" + from typing import Union -from propsutil import get_entry_dict -import pathlib +from envutil import get_proj_dir +from propsutil import get_entry_dict_from_path, update_entry_dict from os import path LANG_FILENAME = 'lastupdated.properties' -def get_last_update_key(language: str) -> str: - return "bundles.{lang}.lastupdated".format({lang=language}) + +def _get_last_update_key(language: str) -> str: + return "bundles.{lang}.lastupdated".format(lang=language) + + +def _get_props_path(): + return path.join(get_proj_dir(), LANG_FILENAME) + def get_commit_for_language(language: str) -> Union[str, None]: - this_path = path.join(get_props_file_path(), LANG_FILENAME) + lang_dict = get_entry_dict_from_path(_get_props_path()) + if lang_dict is None: + return None - if path.isfile(this_path): - lang_dict = get_entry_dict(this_path) - key = get_last_update_key(language) - if key in lang_dict: - return lang_dict[key] + key = _get_last_update_key(language) + if key not in lang_dict: + return None + + return lang_dict[key] - return None def set_commit_for_language(language: str, latest_commit: str): - pass - -def get_props_file_path() -> str: - return str(pathlib.Path(__file__).parent.absolute()) + key = _get_last_update_key(language) + update_entry_dict({key: latest_commit}, _get_props_path()) diff --git a/release_scripts/localization_scripts/lastupdated.properties b/release_scripts/localization_scripts/lastupdated.properties new file mode 100644 index 0000000000..e69de29bb2 diff --git a/release_scripts/localization_scripts/propsutil.py b/release_scripts/localization_scripts/propsutil.py index af48e8173c..dfb5b0e323 100644 --- a/release_scripts/localization_scripts/propsutil.py +++ b/release_scripts/localization_scripts/propsutil.py @@ -37,6 +37,23 @@ def get_entry_dict(file_contents: Union[str, IO]) -> Dict[str, str]: return props.properties +def get_entry_dict_from_path(props_path: str) -> Union[Dict[str, str], None]: + """ + Retrieves a dictionary mapping the properties represented in the string or None if no properties file can be found + at that path. + Args: + props_path: The path to the properties file. + + Returns: The entry dictionary for that properties file. + + """ + if os.path.isfile(props_path): + with open(props_path, "rb") as f: + return get_entry_dict(f) + else: + return None + + def set_entry_dict(contents: Dict[str, str], file_path: str): """Sets the property file to the key-value pairs of the contents dictionary. diff --git a/release_scripts/localization_scripts/updatepropsscript.py b/release_scripts/localization_scripts/updatepropsscript.py index b975a5b115..a1d7bb40f5 100644 --- a/release_scripts/localization_scripts/updatepropsscript.py +++ b/release_scripts/localization_scripts/updatepropsscript.py @@ -6,12 +6,12 @@ from typing import List, Dict, Tuple, Callable, Iterator import sys import os +from envutil import get_proj_dir from langpropsutil import set_commit_for_language from propsutil import set_entry_dict, get_entry_dict, get_lang_bundle_name from csvutil import csv_to_records from propentry import PropEntry import argparse -import pathlib def write_prop_entries(entries: Iterator[PropEntry], repo_path: str): @@ -42,10 +42,8 @@ def update_prop_entries(entries: Iterator[PropEntry], repo_path: str): for rel_path, (entries, to_delete) in items_by_file.items(): abs_path = os.path.join(repo_path, rel_path) - if os.path.isfile(abs_path): - with open(abs_path, "rb") as f: - prop_items = get_entry_dict(f) - else: + prop_items = get_entry_dict(abs_path) + if prop_items is None: prop_items = {} for key_to_delete in to_delete: @@ -184,9 +182,14 @@ def get_new_rel_path(orig_path: str, new_filename: str) -> str: def main(): - parser = argparse.ArgumentParser(description='Updates properties files in the autopsy git repo.') + parser = argparse.ArgumentParser(description='Updates properties files in the autopsy git repo.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument(dest='csv_file', type=str, help='The path to the csv file.') + parser.add_argument(dest='csv_file', type=str, help='The path to the csv file. The default format for the csv ' + 'file has columns of relative path, properties file key, ' + 'properties file value, and commit id for how recent these ' + 'updates are. A header row is expected by default and the ' + 'commit id, if specified, should only be in the first row.') parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, help='The path to the repo. If not specified, path of script is used.') @@ -197,26 +200,26 @@ def main(): parser.add_argument('-v', '--value-idx', dest='value_idx', action='store', type=int, default=2, required=False, help='The column index in the csv file providing the value within the properties file.') parser.add_argument('-d', '--should-delete-idx', dest='should_delete_idx', action='store', type=int, default=None, - required=False, help='The column index in the csv file providing whether or not the file ' + + required=False, help='The column index in the csv file providing whether or not the file ' 'should be deleted. Any non-blank content will be treated as True.') parser.add_argument('-c', '--commit-idx', dest='latest_commit_idx', action='store', type=int, default=3, - required=False, help='The column index in the csv file providing the commit for which this ' + + required=False, help='The column index in the csv file providing the commit for which this ' 'update applies. The commit should be located in the header row. ') parser.add_argument('-f', '--file-rename', dest='file_rename', action='store', type=str, default=None, - required=False, help='If specified, the properties file will be renamed to the argument' + + required=False, help='If specified, the properties file will be renamed to the argument' ' preserving the specified relative path.') parser.add_argument('-z', '--has-no-header', dest='has_no_header', action='store_true', default=False, required=False, help='Specify whether or not there is a header within the csv file.') parser.add_argument('-o', '--should-overwrite', dest='should_overwrite', action='store_true', default=False, - required=False, help="Whether or not to overwrite the previously existing properties files" + + required=False, help="Whether or not to overwrite the previously existing properties files" " ignoring previously existing values.") parser.add_argument('-l', '--language', dest='language', type=str, default='HEAD', required=False, - help='Specify the language in order to update the last updated properties file and rename ' + + help='Specify the language in order to update the last updated properties file and rename ' 'files within directories. This flag overrides the file-rename flag.') args = parser.parse_args() - repo_path = args.repo_path if args.repo_path is not None else str(pathlib.Path(__file__).parent.absolute()) + repo_path = args.repo_path if args.repo_path is not None else get_proj_dir() input_path = args.csv_file path_idx = args.path_idx key_idx = args.key_idx From d8921d2ac4bfbf75a7a2b633f550628210e12b8b Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 14 Jul 2020 08:53:47 -0400 Subject: [PATCH 09/42] add in deletion row expection for update and a readme --- release_scripts/localization_scripts/README.md | 17 +++++++++++++++++ .../localization_scripts/updatepropsscript.py | 16 ++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 release_scripts/localization_scripts/README.md diff --git a/release_scripts/localization_scripts/README.md b/release_scripts/localization_scripts/README.md new file mode 100644 index 0000000000..f2db66fcc8 --- /dev/null +++ b/release_scripts/localization_scripts/README.md @@ -0,0 +1,17 @@ +## Description + +This folder provides tools to handle updates of bundle files for language localization. There are three main scripts: +- `allbundlesscript.py` - generates a csv file containing the relative path of the bundle file, the key, and the value for each property. +- `diffscript.py` - determines the property values that have changed between two commits and generates a csv file containing the relative path, the key, the previous value, the new value, and the change type (addition, deletion, change). +- `updatepropsscript.py` - Given a csv file containing the relative path of the bundle, the key, and the new value, will update the property values for a given language within the project. + +All of these scripts provide more details on usage by calling the script with `-h`. + +## Basic Localization Update Workflow + +1. Call `python3 diffscript.py -l ` to generate a csv file containing differences in properties file values from the language's previous commit to the `HEAD` commit. The language identifier should be the abbreviated identifier used for the bundle (i.e. 'ja' for Japanese). +2. Update csv file with translations +3. Call `python3 updatepropsscript.py -l ` to update properties files based on the newly generated csv file. The csv file should be formatted such that the columns are bundle relative path, property files key, translated value and commit id for the latest commit id for which these changes represent. The commit id only needs to be in the header row. + +## Localization Generation for the First Time +First-time updates should follow a similar procedure except that instead of calling `diffscript.py`, call `python3 allbundlesscript ` to generate a csv file with relative paths of bundle files, property file keys, property file values. \ No newline at end of file diff --git a/release_scripts/localization_scripts/updatepropsscript.py b/release_scripts/localization_scripts/updatepropsscript.py index a1d7bb40f5..289478573c 100644 --- a/release_scripts/localization_scripts/updatepropsscript.py +++ b/release_scripts/localization_scripts/updatepropsscript.py @@ -150,7 +150,7 @@ def get_prop_entries(rows: List[List[str]], def get_should_deleted(row_items: List[str], requested_idx: int) -> bool: - """If there is a value at row_items[requested_idx] and that value is not empty, then this will return true. + """If there is a value at row_items[requested_idx] and that value starts with 'DELET', then this will return true. Args: row_items (List[str]): The row items. @@ -159,7 +159,7 @@ def get_should_deleted(row_items: List[str], requested_idx: int) -> bool: Returns: bool: True if the row specifies it should be deleted. """ - if idx_bounded(requested_idx, len(row_items)) and len((row_items[requested_idx].strip())) > 0: + if idx_bounded(requested_idx, len(row_items)) and row_items[requested_idx].strip().upper().startswith('DELET'): return True else: return False @@ -187,8 +187,10 @@ def main(): parser.add_argument(dest='csv_file', type=str, help='The path to the csv file. The default format for the csv ' 'file has columns of relative path, properties file key, ' - 'properties file value, and commit id for how recent these ' - 'updates are. A header row is expected by default and the ' + 'properties file value, whether or not the key should be ' + 'deleted, and commit id for how recent these updates are. ' + 'If the key should be deleted, the deletion row should be ' + '\'DELETION.\' A header row is expected by default and the ' 'commit id, if specified, should only be in the first row.') parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, @@ -199,12 +201,13 @@ def main(): help='The column index in the csv file providing the key within the properties file.') parser.add_argument('-v', '--value-idx', dest='value_idx', action='store', type=int, default=2, required=False, help='The column index in the csv file providing the value within the properties file.') - parser.add_argument('-d', '--should-delete-idx', dest='should_delete_idx', action='store', type=int, default=None, + parser.add_argument('-d', '--should-delete-idx', dest='should_delete_idx', action='store', type=int, default=3, required=False, help='The column index in the csv file providing whether or not the file ' 'should be deleted. Any non-blank content will be treated as True.') - parser.add_argument('-c', '--commit-idx', dest='latest_commit_idx', action='store', type=int, default=3, + parser.add_argument('-c', '--commit-idx', dest='latest_commit_idx', action='store', type=int, default=4, required=False, help='The column index in the csv file providing the commit for which this ' 'update applies. The commit should be located in the header row. ') + parser.add_argument('-f', '--file-rename', dest='file_rename', action='store', type=str, default=None, required=False, help='If specified, the properties file will be renamed to the argument' ' preserving the specified relative path.') @@ -213,6 +216,7 @@ def main(): parser.add_argument('-o', '--should-overwrite', dest='should_overwrite', action='store_true', default=False, required=False, help="Whether or not to overwrite the previously existing properties files" " ignoring previously existing values.") + parser.add_argument('-l', '--language', dest='language', type=str, default='HEAD', required=False, help='Specify the language in order to update the last updated properties file and rename ' 'files within directories. This flag overrides the file-rename flag.') From f4a73dcc65ec5c71f520bc3f1f6f8441d59b2f58 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 14 Jul 2020 09:33:32 -0400 Subject: [PATCH 10/42] error handling and repo path determining --- .../localization_scripts/allbundlesscript.py | 4 ++-- .../localization_scripts/csvutil.py | 15 +++++++++------ .../localization_scripts/diffscript.py | 4 ++-- .../localization_scripts/gitutil.py | 15 +++++++++++++++ .../localization_scripts/propsutil.py | 18 +++++++++++------- .../localization_scripts/updatepropsscript.py | 9 +++++---- 6 files changed, 44 insertions(+), 21 deletions(-) diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py index 055cc835e8..b9efd7b0b3 100644 --- a/release_scripts/localization_scripts/allbundlesscript.py +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -7,7 +7,7 @@ repo is on correct branch (i.e. develop). import sys from envutil import get_proj_dir -from gitutil import get_property_file_entries, get_commit_id +from gitutil import get_property_file_entries, get_commit_id, get_git_root from csvutil import records_to_csv import argparse @@ -44,7 +44,7 @@ def main(): required=False, help="Suppresses adding commits to the generated csv header.") args = parser.parse_args() - repo_path = args.repo_path if args.repo_path is not None else get_proj_dir() + repo_path = args.repo_path if args.repo_path is not None else get_git_root(get_proj_dir()) output_path = args.output_path show_commit = not args.no_commit diff --git a/release_scripts/localization_scripts/csvutil.py b/release_scripts/localization_scripts/csvutil.py index 4349e781a6..aa382944e1 100644 --- a/release_scripts/localization_scripts/csvutil.py +++ b/release_scripts/localization_scripts/csvutil.py @@ -40,11 +40,14 @@ def csv_to_records(input_path: str, header_row: bool) -> Tuple[List[List[str]], header = None results = [] - for row in csv_reader: - if header_row: - header = row - header_row = False - else: - results.append(row) + try: + for row in csv_reader: + if header_row: + header = row + header_row = False + else: + results.append(row) + except Exception as e: + raise Exception("There was an error parsing csv {path}".format(path=input_path), e) return results, header diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index ebf50dc0c9..b8dfb522c2 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -6,7 +6,7 @@ gitpython and jproperties. As a consequence, it also requires git >= 1.7.0 and import sys from envutil import get_proj_dir -from gitutil import get_property_files_diff, get_commit_id +from gitutil import get_property_files_diff, get_commit_id, get_git_root from itemchange import ItemChange from csvutil import records_to_csv import argparse @@ -59,7 +59,7 @@ def main(): 'Japanese. This flag overrides the first-commit flag.') args = parser.parse_args() - repo_path = args.repo_path if args.repo_path is not None else get_proj_dir() + repo_path = args.repo_path if args.repo_path is not None else get_git_root(get_proj_dir()) output_path = args.output_path commit_1_id = args.commit_1_id if args.language is not None: diff --git a/release_scripts/localization_scripts/gitutil.py b/release_scripts/localization_scripts/gitutil.py index 9b9667d421..333bee1440 100644 --- a/release_scripts/localization_scripts/gitutil.py +++ b/release_scripts/localization_scripts/gitutil.py @@ -13,6 +13,21 @@ def get_text(blob: Blob) -> str: return blob.data_stream.read().decode('utf-8') +def get_git_root(child_path: str) -> str: + """ + Taken from https://stackoverflow.com/questions/22081209/find-the-root-of-the-git-repository-where-the-file-lives, + this obtains the root path of the git repo in which this file exists. + Args: + child_path: The path of a child within the repo. + + Returns: The repo root path. + + """ + git_repo = Repo(child_path, search_parent_directories=True) + git_root = git_repo.git.rev_parse("--show-toplevel") + return git_root + + def get_changed_from_diff(rel_path: str, diff: Diff) -> List[ItemChange]: """Determines changes from a git python diff. diff --git a/release_scripts/localization_scripts/propsutil.py b/release_scripts/localization_scripts/propsutil.py index dfb5b0e323..6d0fd7a5b5 100644 --- a/release_scripts/localization_scripts/propsutil.py +++ b/release_scripts/localization_scripts/propsutil.py @@ -33,7 +33,10 @@ def get_entry_dict(file_contents: Union[str, IO]) -> Dict[str, str]: """ props = Properties() - props.load(file_contents, "utf-8") + try: + props.load(file_contents) + except Exception as e: + raise Exception("There was an error loading properties file {file}".format(file=file_contents), e) return props.properties @@ -84,11 +87,12 @@ def update_entry_dict(contents: Dict[str, str], file_path: str): """ contents_to_edit = contents.copy() - if path.isfile(file_path): - cur_dict = get_entry_dict(file_path) - for cur_key, cur_val in cur_dict.values(): - # only update contents if contents does not already have key - if cur_key not in contents_to_edit: - contents_to_edit[cur_key] = cur_val + cur_dict = get_entry_dict_from_path(file_path) + if cur_dict is None: + cur_dict = {} + for cur_key, cur_val in cur_dict.items(): + # only update contents if contents does not already have key + if cur_key not in contents_to_edit: + contents_to_edit[cur_key] = cur_val set_entry_dict(contents_to_edit, file_path) diff --git a/release_scripts/localization_scripts/updatepropsscript.py b/release_scripts/localization_scripts/updatepropsscript.py index 289478573c..99723a395d 100644 --- a/release_scripts/localization_scripts/updatepropsscript.py +++ b/release_scripts/localization_scripts/updatepropsscript.py @@ -7,8 +7,9 @@ import sys import os from envutil import get_proj_dir +from gitutil import get_git_root from langpropsutil import set_commit_for_language -from propsutil import set_entry_dict, get_entry_dict, get_lang_bundle_name +from propsutil import set_entry_dict, get_entry_dict_from_path, get_lang_bundle_name from csvutil import csv_to_records from propentry import PropEntry import argparse @@ -42,7 +43,7 @@ def update_prop_entries(entries: Iterator[PropEntry], repo_path: str): for rel_path, (entries, to_delete) in items_by_file.items(): abs_path = os.path.join(repo_path, rel_path) - prop_items = get_entry_dict(abs_path) + prop_items = get_entry_dict_from_path(abs_path) if prop_items is None: prop_items = {} @@ -194,7 +195,7 @@ def main(): 'commit id, if specified, should only be in the first row.') parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, - help='The path to the repo. If not specified, path of script is used.') + help='The path to the repo. If not specified, parent repo of path of script is used.') parser.add_argument('-p', '--path-idx', dest='path_idx', action='store', type=int, default=0, required=False, help='The column index in the csv file providing the relative path to the properties file.') parser.add_argument('-k', '--key-idx', dest='key_idx', action='store', type=int, default=1, required=False, @@ -223,7 +224,7 @@ def main(): args = parser.parse_args() - repo_path = args.repo_path if args.repo_path is not None else get_proj_dir() + repo_path = args.repo_path if args.repo_path is not None else get_git_root(get_proj_dir()) input_path = args.csv_file path_idx = args.path_idx key_idx = args.key_idx From d3043afefdf49c1e434a1b71dac31a91ebfb5148 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 14 Jul 2020 11:23:00 -0400 Subject: [PATCH 11/42] fix in readme --- release_scripts/localization_scripts/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release_scripts/localization_scripts/README.md b/release_scripts/localization_scripts/README.md index f2db66fcc8..0b738f8f38 100644 --- a/release_scripts/localization_scripts/README.md +++ b/release_scripts/localization_scripts/README.md @@ -9,9 +9,9 @@ All of these scripts provide more details on usage by calling the script with `- ## Basic Localization Update Workflow -1. Call `python3 diffscript.py -l ` to generate a csv file containing differences in properties file values from the language's previous commit to the `HEAD` commit. The language identifier should be the abbreviated identifier used for the bundle (i.e. 'ja' for Japanese). +1. Call `python3 diffscript.py -l ` to generate a csv file containing differences in properties file values from the language's previous commit to the `HEAD` commit. The language identifier should be the abbreviated identifier used for the bundle (i.e. 'ja' for Japanese). 2. Update csv file with translations -3. Call `python3 updatepropsscript.py -l ` to update properties files based on the newly generated csv file. The csv file should be formatted such that the columns are bundle relative path, property files key, translated value and commit id for the latest commit id for which these changes represent. The commit id only needs to be in the header row. +3. Call `python3 updatepropsscript.py -l ` to update properties files based on the newly generated csv file. The csv file should be formatted such that the columns are bundle relative path, property files key, translated value and commit id for the latest commit id for which these changes represent. The commit id only needs to be in the header row. ## Localization Generation for the First Time First-time updates should follow a similar procedure except that instead of calling `diffscript.py`, call `python3 allbundlesscript ` to generate a csv file with relative paths of bundle files, property file keys, property file values. \ No newline at end of file From 2a23c34b7cbeed082c2cb032a0722aaecce858e7 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 17 Jul 2020 10:06:50 -0400 Subject: [PATCH 12/42] utf-8-sig now for csv read & write; value regex (which needs to be fully implemented) --- .../localization_scripts/allbundlesscript.py | 15 +++++++++--- .../localization_scripts/csvutil.py | 6 ++--- .../localization_scripts/diffscript.py | 24 +++++++++++++++---- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py index b9efd7b0b3..3f062424b8 100644 --- a/release_scripts/localization_scripts/allbundlesscript.py +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -9,10 +9,12 @@ import sys from envutil import get_proj_dir from gitutil import get_property_file_entries, get_commit_id, get_git_root from csvutil import records_to_csv +from typing import Union +import re import argparse -def write_items_to_csv(repo_path: str, output_path: str, show_commit: bool): +def write_items_to_csv(repo_path: str, output_path: str, show_commit: bool, value_regex: Union[str, None]): """Determines the contents of '.properties-MERGED' files and writes to a csv file. Args: @@ -28,7 +30,8 @@ def write_items_to_csv(repo_path: str, output_path: str, show_commit: bool): rows = [row_header] for entry in get_property_file_entries(repo_path): - rows.append([entry.rel_path, entry.key, entry.value]) + if value_regex is None or re.match(value_regex, entry.value): + rows.append([entry.rel_path, entry.key, entry.value]) records_to_csv(output_path, rows) @@ -42,13 +45,19 @@ def main(): help='The path to the repo. If not specified, path of script is used.') parser.add_argument('-nc', '--no_commit', dest='no_commit', action='store_true', default=False, required=False, help="Suppresses adding commits to the generated csv header.") + parser.add_argument('-vr', '--value-regex', dest='value_regex', type=str, default=None, required=False, + help='Specify the regex for the property value where a regex match against the property value ' + 'will display the key value pair in csv output (i.e. \'[a-zA-Z]\' or \'\\S\' for removing ' + 'just whitespace items). If this option is not specified, all key value pairs will be ' + 'accepted.') args = parser.parse_args() repo_path = args.repo_path if args.repo_path is not None else get_git_root(get_proj_dir()) output_path = args.output_path show_commit = not args.no_commit + value_regex = args.value_regex - write_items_to_csv(repo_path, output_path, show_commit) + write_items_to_csv(repo_path, output_path, show_commit, value_regex) sys.exit(0) diff --git a/release_scripts/localization_scripts/csvutil.py b/release_scripts/localization_scripts/csvutil.py index aa382944e1..acfe8e911f 100644 --- a/release_scripts/localization_scripts/csvutil.py +++ b/release_scripts/localization_scripts/csvutil.py @@ -4,6 +4,7 @@ from typing import List, Iterable, Tuple import csv import os +import codecs def records_to_csv(output_path: str, rows: Iterable[List[str]]): @@ -20,9 +21,8 @@ def records_to_csv(output_path: str, rows: Iterable[List[str]]): if not os.path.exists(parent_dir): os.makedirs(parent_dir) - with open(output_path, 'w', encoding="utf-8", newline='') as csvfile: + with open(output_path, 'w', encoding="utf-8-sig", newline='') as csvfile: writer = csv.writer(csvfile) - for row in rows: writer.writerow(row) @@ -35,7 +35,7 @@ def csv_to_records(input_path: str, header_row: bool) -> Tuple[List[List[str]], header_row (bool): Whether or not there is a header row to be skipped. """ - with open(input_path, encoding='utf-8') as csv_file: + with open(input_path, encoding='utf-8-sig') as csv_file: csv_reader = csv.reader(csv_file, delimiter=',') header = None diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index b8dfb522c2..fd4de08093 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -11,11 +11,14 @@ from itemchange import ItemChange from csvutil import records_to_csv import argparse import pathlib +from typing import Union +import re from langpropsutil import get_commit_for_language, LANG_FILENAME -def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit_2_id: str, show_commits: bool): +def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit_2_id: str, show_commits: bool, + value_regex: Union[str, None]): """Determines the changes made in '.properties-MERGED' files from one commit to another commit. Args: @@ -23,7 +26,9 @@ def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit output_path (str): The output path for the csv file. commit_1_id (str): The initial commit for the diff. commit_2_id (str): The latest commit for the diff. - show_commits (bool): show commits in the header row. + show_commits (bool): Show commits in the header row. + value_regex (Union[str, None]): If non-none, only key value pairs where the value is a regex match with this + value will be included. """ row_header = ItemChange.get_headers() @@ -32,8 +37,11 @@ def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit rows = [row_header] - rows += map(lambda item_change: item_change.get_row(), - get_property_files_diff(repo_path, commit_1_id, commit_2_id)) + item_changes = get_property_files_diff(repo_path, commit_1_id, commit_2_id) + if value_regex is not None: + item_changes = filter(lambda item_change: re.match(value_regex, item_change.cur_val) is not None, item_changes) + + rows += map(lambda item_change: item_change.get_row(), item_changes) records_to_csv(output_path, rows) @@ -57,11 +65,17 @@ def main(): parser.add_argument('-l', '--language', dest='language', type=str, default='HEAD', required=False, help='Specify the language in order to determine the first commit to use (i.e. \'ja\' for ' 'Japanese. This flag overrides the first-commit flag.') + parser.add_argument('-vr', '--value-regex', dest='value_regex', type=str, default=None, required=False, + help='Specify the regex for the property value where a regex match against the property value ' + 'will display the key value pair in csv output (i.e. \'[a-zA-Z]\' or \'\\S\' for removing ' + 'just whitespace items). If this option is not specified, all key value pairs will be ' + 'accepted.') args = parser.parse_args() repo_path = args.repo_path if args.repo_path is not None else get_git_root(get_proj_dir()) output_path = args.output_path commit_1_id = args.commit_1_id + value_regex = args.value_regex if args.language is not None: commit_1_id = get_commit_for_language(args.language) @@ -74,7 +88,7 @@ def main(): commit_2_id = args.commit_2_id show_commits = not args.no_commits - write_diff_to_csv(repo_path, output_path, commit_1_id, commit_2_id, show_commits) + write_diff_to_csv(repo_path, output_path, commit_1_id, commit_2_id, show_commits, value_regex) sys.exit(0) From 526876a64f950b6f638296b7cf89058268620bbf Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 17 Jul 2020 10:19:44 -0400 Subject: [PATCH 13/42] some code cleanup updates --- release_scripts/localization_scripts/allbundlesscript.py | 3 +++ release_scripts/localization_scripts/csvutil.py | 1 - release_scripts/localization_scripts/diffscript.py | 4 ++-- release_scripts/localization_scripts/propsutil.py | 1 - release_scripts/localization_scripts/updatepropsscript.py | 1 + 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py index 3f062424b8..8e2ea6303d 100644 --- a/release_scripts/localization_scripts/allbundlesscript.py +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -21,6 +21,8 @@ def write_items_to_csv(repo_path: str, output_path: str, show_commit: bool, valu repo_path (str): The local path to the git repo. output_path (str): The output path for the csv file. show_commit (bool): Whether or not to include the commit id in the header + value_regex (Union[str, None]): If non-none, only key value pairs where the value is a regex match with this + value will be included. """ row_header = ['Relative path', 'Key', 'Value'] @@ -37,6 +39,7 @@ def write_items_to_csv(repo_path: str, output_path: str, show_commit: bool, valu def main(): + # noinspection PyTypeChecker parser = argparse.ArgumentParser(description='Gathers all key-value pairs within .properties-MERGED files into ' 'one csv file.', formatter_class=argparse.ArgumentDefaultsHelpFormatter) diff --git a/release_scripts/localization_scripts/csvutil.py b/release_scripts/localization_scripts/csvutil.py index acfe8e911f..7ee99eccd8 100644 --- a/release_scripts/localization_scripts/csvutil.py +++ b/release_scripts/localization_scripts/csvutil.py @@ -4,7 +4,6 @@ from typing import List, Iterable, Tuple import csv import os -import codecs def records_to_csv(output_path: str, rows: Iterable[List[str]]): diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index fd4de08093..9c29533081 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -10,7 +10,6 @@ from gitutil import get_property_files_diff, get_commit_id, get_git_root from itemchange import ItemChange from csvutil import records_to_csv import argparse -import pathlib from typing import Union import re @@ -47,7 +46,8 @@ def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit def main(): - parser = argparse.ArgumentParser(description="determines the updated, added, and deleted properties from the " + # noinspection PyTypeChecker + parser = argparse.ArgumentParser(description="Determines the updated, added, and deleted properties from the " "'.properties-MERGED' files and generates a csv file containing " "the items changed.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) diff --git a/release_scripts/localization_scripts/propsutil.py b/release_scripts/localization_scripts/propsutil.py index 6d0fd7a5b5..de94a82271 100644 --- a/release_scripts/localization_scripts/propsutil.py +++ b/release_scripts/localization_scripts/propsutil.py @@ -4,7 +4,6 @@ from typing import Dict, Union, IO from jproperties import Properties import os -from os import path # The default extension for property files in autopsy repo DEFAULT_PROPS_EXTENSION = 'properties-MERGED' diff --git a/release_scripts/localization_scripts/updatepropsscript.py b/release_scripts/localization_scripts/updatepropsscript.py index 99723a395d..edce1aafeb 100644 --- a/release_scripts/localization_scripts/updatepropsscript.py +++ b/release_scripts/localization_scripts/updatepropsscript.py @@ -183,6 +183,7 @@ def get_new_rel_path(orig_path: str, new_filename: str) -> str: def main(): + # noinspection PyTypeChecker parser = argparse.ArgumentParser(description='Updates properties files in the autopsy git repo.', formatter_class=argparse.ArgumentDefaultsHelpFormatter) From e4c20b129be617f1ae123f7923f79a825ad072b1 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 17 Jul 2020 11:03:32 -0400 Subject: [PATCH 14/42] refinement --- .../localization_scripts/itemchange.py | 6 ++++++ .../localization_scripts/propentry.py | 5 +++++ .../localization_scripts/regexutil.py | 16 ++++++++++++++++ 3 files changed, 27 insertions(+) create mode 100644 release_scripts/localization_scripts/regexutil.py diff --git a/release_scripts/localization_scripts/itemchange.py b/release_scripts/localization_scripts/itemchange.py index 8533be9469..698e710375 100644 --- a/release_scripts/localization_scripts/itemchange.py +++ b/release_scripts/localization_scripts/itemchange.py @@ -3,6 +3,12 @@ from propsutil import get_entry_dict class ItemChange: + rel_path: str + key: str + prev_val: str + cur_val: str + type: str + def __init__(self, rel_path: str, key: str, prev_val: str, cur_val: str): """Describes the change that occurred for a particular key of a properties file. diff --git a/release_scripts/localization_scripts/propentry.py b/release_scripts/localization_scripts/propentry.py index 4821f4bc5f..d6930c1c1e 100644 --- a/release_scripts/localization_scripts/propentry.py +++ b/release_scripts/localization_scripts/propentry.py @@ -1,4 +1,9 @@ class PropEntry: + rel_path: str + key: str + value: str + should_delete: str + def __init__(self, rel_path: str, key: str, value: str, should_delete: bool = False): """Defines a property file entry to be updated in a property file. diff --git a/release_scripts/localization_scripts/regexutil.py b/release_scripts/localization_scripts/regexutil.py new file mode 100644 index 0000000000..ce1952e3bb --- /dev/null +++ b/release_scripts/localization_scripts/regexutil.py @@ -0,0 +1,16 @@ +from typing import TypeVar, Callable, Tuple + + +def is_match(content: str, regex: str) -> bool: + pass + +T = TypeVar('T') + +def is_match_on_field(obj: T, prop_retriever: Callable[T, str]) -> bool: + pass + +class SeparationResult: + + def __init__(self, ): + +def separate(obj: T, prop_retriever: Callable[T, str]) -> From 2c5bab645c5ed29a5e2befe605dc61b8729802b3 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 21 Jul 2020 08:28:39 -0400 Subject: [PATCH 15/42] omitted files work --- .../localization_scripts/allbundlesscript.py | 14 ++++++++--- .../localization_scripts/diffscript.py | 25 +++++++++++-------- .../localization_scripts/itemchange.py | 16 +++++++++--- .../localization_scripts/regexutil.py | 16 ------------ .../localization_scripts/updatepropsscript.py | 21 +++------------- 5 files changed, 40 insertions(+), 52 deletions(-) delete mode 100644 release_scripts/localization_scripts/regexutil.py diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py index 8e2ea6303d..b699ee5d89 100644 --- a/release_scripts/localization_scripts/allbundlesscript.py +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -7,6 +7,7 @@ repo is on correct branch (i.e. develop). import sys from envutil import get_proj_dir +from fileutil import get_filename_addition, OMITTED_ADDITION from gitutil import get_property_file_entries, get_commit_id, get_git_root from csvutil import records_to_csv from typing import Union @@ -29,13 +30,20 @@ def write_items_to_csv(repo_path: str, output_path: str, show_commit: bool, valu if show_commit: row_header.append(get_commit_id(repo_path, 'HEAD')) - rows = [row_header] + rows = [] + omitted = [] for entry in get_property_file_entries(repo_path): + new_entry = [entry.rel_path, entry.key, entry.value] if value_regex is None or re.match(value_regex, entry.value): - rows.append([entry.rel_path, entry.key, entry.value]) + rows.append(new_entry) + else: + omitted.append(new_entry) - records_to_csv(output_path, rows) + records_to_csv(output_path, [row_header] + rows) + + if len(omitted) > 0: + records_to_csv(get_filename_addition(output_path, OMITTED_ADDITION), [row_header] + omitted) def main(): diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index 9c29533081..6db186e2fc 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -2,17 +2,15 @@ and generates a csv file containing the items changed. This script requires the python libraries: gitpython and jproperties. As a consequence, it also requires git >= 1.7.0 and python >= 3.4. """ - +import re import sys - from envutil import get_proj_dir +from fileutil import get_filename_addition, OMITTED_ADDITION from gitutil import get_property_files_diff, get_commit_id, get_git_root -from itemchange import ItemChange +from itemchange import ItemChange, ChangeType from csvutil import records_to_csv import argparse from typing import Union -import re - from langpropsutil import get_commit_for_language, LANG_FILENAME @@ -34,15 +32,20 @@ def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit if show_commits: row_header += [get_commit_id(repo_path, commit_1_id), get_commit_id(repo_path, commit_2_id)] - rows = [row_header] + rows = [] + omitted = [] - item_changes = get_property_files_diff(repo_path, commit_1_id, commit_2_id) - if value_regex is not None: - item_changes = filter(lambda item_change: re.match(value_regex, item_change.cur_val) is not None, item_changes) + for entry in get_property_files_diff(repo_path, commit_1_id, commit_2_id): + new_entry = [entry.rel_path, entry.key, entry.value] + if value_regex is not None and (entry.type == ChangeType.DELETION or (not re.match(value_regex, entry.value))): + omitted.append(new_entry) + else: + rows.append(new_entry) - rows += map(lambda item_change: item_change.get_row(), item_changes) + records_to_csv(output_path, [row_header] + rows) - records_to_csv(output_path, rows) + if len(omitted) > 0: + records_to_csv(get_filename_addition(output_path, OMITTED_ADDITION), [row_header] + omitted) def main(): diff --git a/release_scripts/localization_scripts/itemchange.py b/release_scripts/localization_scripts/itemchange.py index 698e710375..948ac55175 100644 --- a/release_scripts/localization_scripts/itemchange.py +++ b/release_scripts/localization_scripts/itemchange.py @@ -1,5 +1,13 @@ from typing import Iterator, List, Union from propsutil import get_entry_dict +from enum import Enum + + +class ChangeType(Enum): + """Describes the nature of a change in the properties file.""" + ADDITION = 'ADDITION' + DELETION = 'DELETION' + CHANGE = 'CHANGE' class ItemChange: @@ -7,7 +15,7 @@ class ItemChange: key: str prev_val: str cur_val: str - type: str + type: ChangeType def __init__(self, rel_path: str, key: str, prev_val: str, cur_val: str): """Describes the change that occurred for a particular key of a properties file. @@ -23,11 +31,11 @@ class ItemChange: self.prev_val = prev_val self.cur_val = cur_val if ItemChange.has_str_content(cur_val) and not ItemChange.has_str_content(prev_val): - self.type = 'ADDITION' + self.type = ChangeType.ADDITION elif not ItemChange.has_str_content(cur_val) and ItemChange.has_str_content(prev_val): - self.type = 'DELETION' + self.type = ChangeType.DELETION else: - self.type = 'CHANGE' + self.type = ChangeType.CHANGE @staticmethod def has_str_content(content: str): diff --git a/release_scripts/localization_scripts/regexutil.py b/release_scripts/localization_scripts/regexutil.py deleted file mode 100644 index ce1952e3bb..0000000000 --- a/release_scripts/localization_scripts/regexutil.py +++ /dev/null @@ -1,16 +0,0 @@ -from typing import TypeVar, Callable, Tuple - - -def is_match(content: str, regex: str) -> bool: - pass - -T = TypeVar('T') - -def is_match_on_field(obj: T, prop_retriever: Callable[T, str]) -> bool: - pass - -class SeparationResult: - - def __init__(self, ): - -def separate(obj: T, prop_retriever: Callable[T, str]) -> diff --git a/release_scripts/localization_scripts/updatepropsscript.py b/release_scripts/localization_scripts/updatepropsscript.py index edce1aafeb..2663ee330a 100644 --- a/release_scripts/localization_scripts/updatepropsscript.py +++ b/release_scripts/localization_scripts/updatepropsscript.py @@ -7,6 +7,7 @@ import sys import os from envutil import get_proj_dir +from fileutil import get_new_path from gitutil import get_git_root from langpropsutil import set_commit_for_language from propsutil import set_entry_dict, get_entry_dict_from_path, get_lang_bundle_name @@ -166,22 +167,6 @@ def get_should_deleted(row_items: List[str], requested_idx: int) -> bool: return False -def get_new_rel_path(orig_path: str, new_filename: str) -> str: - """Obtains a new relative path. This tries to determine if the provided path is a directory or filename (has an - extension containing '.') then constructs the new path with the old parent directory and the new filename. - - Args: - orig_path (str): The original path. - new_filename (str): The new filename to use. - - Returns: - str: The new path. - """ - potential_parent_dir, orig_file = os.path.split(orig_path) - parent_dir = orig_path if '.' not in orig_file else potential_parent_dir - return os.path.join(parent_dir, new_filename) - - def main(): # noinspection PyTypeChecker parser = argparse.ArgumentParser(description='Updates properties files in the autopsy git repo.', @@ -243,10 +228,10 @@ def main(): # provides the means of renaming the bundle file if args.language is not None: def path_converter(orig_path: str): - return get_new_rel_path(orig_path, get_lang_bundle_name(args.language)) + return get_new_path(orig_path, get_lang_bundle_name(args.language)) elif args.file_rename is not None: def path_converter(orig_path: str): - return get_new_rel_path(orig_path, args.file_rename) + return get_new_path(orig_path, args.file_rename) else: path_converter = None From 7d3b36f44ee6f28e8969f33074fd691c1ee4f66c Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 21 Jul 2020 08:28:49 -0400 Subject: [PATCH 16/42] omitted files work --- .../localization_scripts/fileutil.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 release_scripts/localization_scripts/fileutil.py diff --git a/release_scripts/localization_scripts/fileutil.py b/release_scripts/localization_scripts/fileutil.py new file mode 100644 index 0000000000..e7991a8209 --- /dev/null +++ b/release_scripts/localization_scripts/fileutil.py @@ -0,0 +1,60 @@ +import os +from typing import Union, Tuple + + +def get_path_pieces(orig_path: str) -> Tuple[str, Union[str, None], Union[str, None]]: + """Retrieves path pieces. This is a naive approach as it determines if a file is present based on the + presence of an extension. + Args: + orig_path: The original path to deconstruct. + + Returns: A tuple of directory, filename and extension. If no extension is present, filename and extension are None. + + """ + + potential_parent_dir, orig_file = os.path.split(orig_path) + filename, file_extension = os.path.splitext(orig_file) + + if file_extension is None or len(file_extension) < 1: + return orig_path, None, None + else: + return potential_parent_dir, filename, file_extension + + +def get_new_path(orig_path: str, new_filename: str) -> str: + """Obtains a new path. This tries to determine if the provided path is a directory or filename (has an + extension containing '.') then constructs the new path with the old parent directory and the new filename. + + Args: + orig_path (str): The original path. + new_filename (str): The new filename to use. + + Returns: + str: The new path. + """ + + parent_dir, filename, ext = get_path_pieces(orig_path) + return os.path.join(parent_dir, new_filename) + + +# For use with creating csv filenames for entries that have been omitted. +OMITTED_ADDITION = '-omitted' + + +def get_filename_addition(orig_path: str, filename_addition: str) -> str: + """Gets filename with addition. So if item is '/path/name.ext' and the filename_addition is '-add', the new result + would be '/path/name-add.ext'. + + Args: + orig_path (str): The original path. + filename_addition (str): The new addition. + + Returns: The altered path. + + """ + parent_dir, filename, extension = get_path_pieces(orig_path) + if filename is None: + return orig_path + filename_addition + else: + ext = '' if extension is None else extension + return os.path.join(parent_dir, '{0}{1}.{2}'.format(filename, filename_addition, ext)) From 4aaed340bdea87c0461ba07e597ab6f6e9a8cff3 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 21 Jul 2020 16:11:10 -0400 Subject: [PATCH 17/42] some small bug fixes and comments --- release_scripts/localization_scripts/allbundlesscript.py | 3 ++- release_scripts/localization_scripts/diffscript.py | 8 +++++--- release_scripts/localization_scripts/propentry.py | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py index b699ee5d89..71146ffc91 100644 --- a/release_scripts/localization_scripts/allbundlesscript.py +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -60,7 +60,8 @@ def main(): help='Specify the regex for the property value where a regex match against the property value ' 'will display the key value pair in csv output (i.e. \'[a-zA-Z]\' or \'\\S\' for removing ' 'just whitespace items). If this option is not specified, all key value pairs will be ' - 'accepted.') + 'accepted. If this option is specified, another csv file will be created in the same ' + 'directory as the original output_path with \'' + OMITTED_ADDITION + ' appended.') args = parser.parse_args() repo_path = args.repo_path if args.repo_path is not None else get_git_root(get_proj_dir()) diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index 6db186e2fc..df805d7236 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -36,8 +36,8 @@ def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit omitted = [] for entry in get_property_files_diff(repo_path, commit_1_id, commit_2_id): - new_entry = [entry.rel_path, entry.key, entry.value] - if value_regex is not None and (entry.type == ChangeType.DELETION or (not re.match(value_regex, entry.value))): + new_entry = entry.get_row() + if value_regex is not None and (entry.type == ChangeType.DELETION or not re.match(value_regex, entry.cur_val)): omitted.append(new_entry) else: rows.append(new_entry) @@ -72,7 +72,9 @@ def main(): help='Specify the regex for the property value where a regex match against the property value ' 'will display the key value pair in csv output (i.e. \'[a-zA-Z]\' or \'\\S\' for removing ' 'just whitespace items). If this option is not specified, all key value pairs will be ' - 'accepted.') + 'accepted. If this option is specified, entries marked as \'DELETION\' will also be ' + 'omitted another csv file will be created in the same directory as the original ' + 'output_path with \'' + OMITTED_ADDITION + ' appended.') args = parser.parse_args() repo_path = args.repo_path if args.repo_path is not None else get_git_root(get_proj_dir()) diff --git a/release_scripts/localization_scripts/propentry.py b/release_scripts/localization_scripts/propentry.py index d6930c1c1e..99c00f749e 100644 --- a/release_scripts/localization_scripts/propentry.py +++ b/release_scripts/localization_scripts/propentry.py @@ -2,7 +2,7 @@ class PropEntry: rel_path: str key: str value: str - should_delete: str + should_delete: bool def __init__(self, rel_path: str, key: str, value: str, should_delete: bool = False): """Defines a property file entry to be updated in a property file. From f9be107e2501a74a2d08436a181aa5b2ef6608ca Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 21 Jul 2020 21:05:01 -0400 Subject: [PATCH 18/42] 6531 allow users manual adjustments to size of details area to be preserved --- .../discovery/DiscoveryTopComponent.java | 34 ++++++++++++++++--- .../autopsy/discovery/ResultsPanel.form | 4 +-- .../autopsy/discovery/ResultsPanel.java | 4 +-- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/discovery/DiscoveryTopComponent.java b/Core/src/org/sleuthkit/autopsy/discovery/DiscoveryTopComponent.java index 526207981c..56954a2be3 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/DiscoveryTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/DiscoveryTopComponent.java @@ -22,6 +22,8 @@ import com.google.common.eventbus.Subscribe; import java.awt.BorderLayout; import java.awt.Color; import java.awt.Graphics; +import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; import java.util.List; import java.util.stream.Collectors; import javax.swing.JSplitPane; @@ -53,7 +55,8 @@ public final class DiscoveryTopComponent extends TopComponent { private int dividerLocation = -1; private static final int ANIMATION_INCREMENT = 10; - private static final int RESULTS_AREA_SMALL_SIZE = 250; + private volatile static int resultsAreaSize = 250; + private volatile static int previousResultsAreaSize = -1; private SwingAnimator animator = null; @@ -78,6 +81,23 @@ public final class DiscoveryTopComponent extends TopComponent { } }); + rightSplitPane.addPropertyChangeListener(JSplitPane.DIVIDER_LOCATION_PROPERTY, new PropertyChangeListener() { + @Override + public void propertyChange(PropertyChangeEvent evt) { + if (evt.getPropertyName().equalsIgnoreCase(JSplitPane.DIVIDER_LOCATION_PROPERTY)) { + //Only change the saved location when it was a manual change by the user and not the animation or the window opening initially + if ((animator == null || !animator.isRunning()) && evt.getNewValue() instanceof Integer + && ((int) evt.getNewValue() + 5) < (rightSplitPane.getHeight() - rightSplitPane.getDividerSize()) && (int) evt.getNewValue() != previousResultsAreaSize) { + System.out.println("setting small size" + (int) evt.getNewValue()); + System.out.println("HEIGHT: " + rightSplitPane.getHeight()); + System.out.println("DIVIDER LOC: " + rightSplitPane.getDividerLocation()); + previousResultsAreaSize = (int) evt.getOldValue(); + resultsAreaSize = (int) evt.getNewValue(); + + } + } + } + }); } /** @@ -245,6 +265,7 @@ public final class DiscoveryTopComponent extends TopComponent { void handleDetailsVisibleEvent(DiscoveryEventUtils.DetailsVisibleEvent detailsVisibleEvent) { if (animator != null && animator.isRunning()) { animator.stop(); + animator = null; } dividerLocation = rightSplitPane.getDividerLocation(); if (detailsVisibleEvent.isShowDetailsArea()) { @@ -316,8 +337,9 @@ public final class DiscoveryTopComponent extends TopComponent { @Override public boolean hasTerminated() { - if (dividerLocation != JSplitPane.UNDEFINED_CONDITION && dividerLocation < RESULTS_AREA_SMALL_SIZE) { - dividerLocation = RESULTS_AREA_SMALL_SIZE; + if (dividerLocation != JSplitPane.UNDEFINED_CONDITION && dividerLocation < resultsAreaSize) { + dividerLocation = resultsAreaSize; + animator = null; return true; } return false; @@ -340,6 +362,7 @@ public final class DiscoveryTopComponent extends TopComponent { public boolean hasTerminated() { if (dividerLocation > rightSplitPane.getHeight() || dividerLocation == JSplitPane.UNDEFINED_CONDITION) { dividerLocation = rightSplitPane.getHeight(); + animator = null; return true; } return false; @@ -362,8 +385,9 @@ public final class DiscoveryTopComponent extends TopComponent { @Override public void paintComponent(Graphics g) { - if ((dividerLocation == JSplitPane.UNDEFINED_CONDITION) || (dividerLocation <= rightSplitPane.getHeight() && dividerLocation >= RESULTS_AREA_SMALL_SIZE)) { - rightSplitPane.setDividerLocation(dividerLocation); + if (animator != null && animator.isRunning() && (dividerLocation == JSplitPane.UNDEFINED_CONDITION + || (dividerLocation <= getHeight() && dividerLocation >= resultsAreaSize))) { + setDividerLocation(dividerLocation); } super.paintComponent(g); } diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ResultsPanel.form b/Core/src/org/sleuthkit/autopsy/discovery/ResultsPanel.form index d35521df46..cad2e006cd 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/ResultsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/discovery/ResultsPanel.form @@ -3,7 +3,7 @@
- + @@ -315,7 +315,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ResultsPanel.java b/Core/src/org/sleuthkit/autopsy/discovery/ResultsPanel.java index 7082b0d835..fc86f7a963 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/ResultsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/ResultsPanel.java @@ -376,7 +376,7 @@ final class ResultsPanel extends javax.swing.JPanel { javax.swing.Box.Filler filler4 = new javax.swing.Box.Filler(new java.awt.Dimension(0, 0), new java.awt.Dimension(0, 0), new java.awt.Dimension(32767, 0)); resultsViewerPanel = new javax.swing.JPanel(); - setMinimumSize(new java.awt.Dimension(700, 200)); + setMinimumSize(new java.awt.Dimension(300, 60)); setPreferredSize(new java.awt.Dimension(700, 700)); setLayout(new java.awt.BorderLayout()); @@ -533,7 +533,7 @@ final class ResultsPanel extends javax.swing.JPanel { add(pagingPanel, java.awt.BorderLayout.PAGE_START); - resultsViewerPanel.setMinimumSize(new java.awt.Dimension(0, 160)); + resultsViewerPanel.setMinimumSize(new java.awt.Dimension(0, 60)); resultsViewerPanel.setPreferredSize(new java.awt.Dimension(700, 700)); resultsViewerPanel.setLayout(new java.awt.BorderLayout()); add(resultsViewerPanel, java.awt.BorderLayout.CENTER); From aa22f3d9afe2cbfe0825f3bb8c6eae3f7fdf4792 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 22 Jul 2020 10:44:53 -0400 Subject: [PATCH 19/42] testing --- .../localization_scripts/csvutil.py | 44 ++++++++- .../localization_scripts/fileutil.py | 46 +++++++++ .../localization_scripts/itemchange.py | 94 ++++++++++++++++++- 3 files changed, 177 insertions(+), 7 deletions(-) diff --git a/release_scripts/localization_scripts/csvutil.py b/release_scripts/localization_scripts/csvutil.py index 7ee99eccd8..eb59b42ad4 100644 --- a/release_scripts/localization_scripts/csvutil.py +++ b/release_scripts/localization_scripts/csvutil.py @@ -1,9 +1,12 @@ """Provides tools for parsing and writing to a csv file. """ - -from typing import List, Iterable, Tuple +import codecs +from typing import List, Iterable, Tuple, TypeVar import csv import os +import unittest +from envutil import get_proj_dir +from unittestutil import TEST_OUTPUT_FOLDER def records_to_csv(output_path: str, rows: Iterable[List[str]]): @@ -50,3 +53,40 @@ def csv_to_records(input_path: str, header_row: bool) -> Tuple[List[List[str]], raise Exception("There was an error parsing csv {path}".format(path=input_path), e) return results, header + + +class CsvUtilTest(unittest.TestCase): + T = TypeVar('T') + + def assert_equal_arr(self, a: List[T], b: List[T]): + self.assertEqual(len(a), len(b), 'arrays are not equal length') + for i in range(0, len(a)): + if isinstance(a[i], list) and isinstance(b[i], list): + self.assert_equal_arr(a[i], b[i]) + else: + self.assertEqual(a[i], b[i], "Items: {0} and {1} at index {2} are not equal.".format(a[i], b[i], i)) + + def test_read_write(self): + data = [['header1', 'header2', 'header3', 'additional header'], + ['data1', 'data2', 'data3'], + ['', 'data2-1', 'data2-2']] + + os.makedirs(os.path.join(get_proj_dir(), TEST_OUTPUT_FOLDER)) + test_path = os.path.join(get_proj_dir(), TEST_OUTPUT_FOLDER, 'test.csv') + records_to_csv(test_path, data) + + byte_inf = min(32, os.path.getsize(test_path)) + with open(test_path, 'rb') as bom_test_file: + raw = bom_test_file.read(byte_inf) + if not raw.startswith(codecs.BOM_UTF8): + self.fail("written csv does not have appropriate BOM") + + read_records_no_header, no_header = csv_to_records(test_path, header_row=False) + self.assert_equal_arr(read_records_no_header, data) + + read_rows, header = csv_to_records(test_path, header_row=True) + self.assert_equal_arr(header, data[0]) + self.assert_equal_arr(read_rows, [data[1], data[2]]) + + + diff --git a/release_scripts/localization_scripts/fileutil.py b/release_scripts/localization_scripts/fileutil.py index e7991a8209..6554b94270 100644 --- a/release_scripts/localization_scripts/fileutil.py +++ b/release_scripts/localization_scripts/fileutil.py @@ -1,4 +1,5 @@ import os +import unittest from typing import Union, Tuple @@ -58,3 +59,48 @@ def get_filename_addition(orig_path: str, filename_addition: str) -> str: else: ext = '' if extension is None else extension return os.path.join(parent_dir, '{0}{1}.{2}'.format(filename, filename_addition, ext)) + + +class FileUtilTest(unittest.TestCase): + + @staticmethod + def _joined_paths(pieces: Tuple[str, str, str]) -> str: + return os.path.join(pieces[0], pieces[1] + '.' + pieces[2]) + + PATH_PIECES1 = ('/test/folder', 'filename', 'ext') + PATH_PIECES2 = ('/test.test2/folder.test2', 'filename.test', 'ext') + PATH_PIECES3 = ('/test.test2/folder.test2/folder', None, None) + + def __init__(self): + self.PATH1 = FileUtilTest._joined_paths(self.PATH_PIECES1) + self.PATH2 = FileUtilTest._joined_paths(self.PATH_PIECES2) + self.PATH3 = self.PATH_PIECES3[0] + + self.ALL_ITEMS = [ + (self.PATH_PIECES1, self.PATH1), + (self.PATH_PIECES2, self.PATH2), + (self.PATH_PIECES3, self.PATH3) + ] + + def get_path_pieces_test(self): + for (expected_path, expected_filename, expected_ext), path in self.ALL_ITEMS: + path, filename, ext = get_path_pieces(path) + self.assertEqual(path, expected_path) + self.assertEqual(filename, expected_filename) + self.assertEqual(ext, expected_ext) + + def get_new_path_test(self): + for (expected_path, expected_filename, expected_ext), path in self.ALL_ITEMS: + new_name = "newname.file" + new_path = get_new_path(path, new_name) + self.assertEqual(new_path, os.path.join(expected_path, new_name)) + + def get_filename_addition_test(self): + for (expected_path, expected_filename, expected_ext), path in self.ALL_ITEMS: + addition = "addition" + new_path = get_filename_addition(path, addition) + self.assertEqual( + new_path, os.path.join( + expected_path, + "{file_name}{addition}.{extension}".format( + file_name=expected_filename, addition=addition, extension=expected_ext))) diff --git a/release_scripts/localization_scripts/itemchange.py b/release_scripts/localization_scripts/itemchange.py index 948ac55175..bfc09b728a 100644 --- a/release_scripts/localization_scripts/itemchange.py +++ b/release_scripts/localization_scripts/itemchange.py @@ -1,4 +1,5 @@ -from typing import Iterator, List, Union +import unittest +from typing import Iterator, List, Union, Dict from propsutil import get_entry_dict from enum import Enum @@ -13,8 +14,8 @@ class ChangeType(Enum): class ItemChange: rel_path: str key: str - prev_val: str - cur_val: str + prev_val: Union[str, None] + cur_val: Union[str, None] type: ChangeType def __init__(self, rel_path: str, key: str, prev_val: str, cur_val: str): @@ -91,7 +92,9 @@ def get_item_change(rel_path: str, key: str, prev_val: str, cur_val: str) -> Uni def get_changed(rel_path: str, a_str: str, b_str: str) -> Iterator[ItemChange]: - """Given the relative path of the properties file that + """Given the relative path of the properties file that has been provided, + determines the property items that have changed between the two property + file strings. Args: rel_path (str): The relative path for the properties file. @@ -101,10 +104,91 @@ def get_changed(rel_path: str, a_str: str, b_str: str) -> Iterator[ItemChange]: Returns: List[ItemChange]: The changes determined. """ - print('Retrieving changes for {}...'.format(rel_path)) + print('Retrieving changes for {0}...'.format(rel_path)) a_dict = get_entry_dict(a_str) b_dict = get_entry_dict(b_str) all_keys = set().union(a_dict.keys(), b_dict.keys()) mapped = map(lambda key: get_item_change( rel_path, key, a_dict.get(key), b_dict.get(key)), all_keys) return filter(lambda entry: entry is not None, mapped) + + +class ItemChangeTest(unittest.TestCase): + @staticmethod + def dict_to_prop_str(dict: Dict[str,str]) -> str: + toret = '' + for key,val in dict.items: + toret += "{key}={value}\n".format(key=key,val=val) + + return toret + + def get_changed_test(self): + deleted_key = 'deleted.property.key' + deleted_val = 'will be deleted' + + change_key = 'change.property.key' + change_val_a = 'original value' + change_val_b = 'new value' + + change_key2 = 'change2.property.key' + change_val2_a = 'original value 2' + change_val2_b = '' + + addition_key = 'addition.property.key' + addition_new_val = 'the added value' + + same_key = 'samevalue.property.key' + same_value = 'the same value' + + same_key2 = 'samevalue2.property.key' + same_value2 = '' + + a_dict = { + deleted_key: deleted_val, + change_key: change_val_a, + change_key2: change_val2_a, + same_key: same_value, + same_key2: same_value2 + } + + b_dict = { + change_key: change_val_b, + change_key2: change_val2_b, + addition_key: addition_new_val, + same_key: same_value, + same_key2: same_value2 + } + + a_str = ItemChangeTest.dict_to_prop_str(a_dict) + b_str = ItemChangeTest.dict_to_prop_str(b_dict) + + rel_path = 'my/rel/path.properties' + + key_to_change = {} + + for item_change in get_changed(rel_path, a_str, b_str): + self.assertEqual(item_change.rel_path, rel_path) + key_to_change[item_change.key] = item_change + + deleted_item = key_to_change[deleted_key] + self.assertEqual(deleted_item.type, ChangeType.DELETION) + self.assertEqual(deleted_item.prev_val, deleted_val) + self.assertEqual(deleted_item.cur_val, None) + + addition_item = key_to_change[addition_key] + self.assertEqual(addition_item.type, ChangeType.ADDITION) + self.assertEqual(addition_item.prev_val, None) + self.assertEqual(deleted_item.cur_val, addition_new_val) + + change_item = key_to_change[change_key] + self.assertEqual(change_item.type, ChangeType.CHANGE) + self.assertEqual(change_item.prev_val, change_val_a) + self.assertEqual(change_item.cur_val, change_val_b) + + change_item2 = key_to_change[change_key2] + self.assertEqual(change_item2.type, ChangeType.CHANGE) + self.assertEqual(change_item2.prev_val, change_val2_a) + self.assertEqual(change_item2.cur_val, change_val2_b) + + self.assertTrue(same_key not in key_to_change) + self.assertTrue(same_key2 not in key_to_change) \ No newline at end of file From aab740e57e939619a4480a4e3b4f833dfd263fb3 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 22 Jul 2020 10:45:17 -0400 Subject: [PATCH 20/42] testing --- release_scripts/localization_scripts/testartifacts/.gitignore | 1 + release_scripts/localization_scripts/unittestutil.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 release_scripts/localization_scripts/testartifacts/.gitignore create mode 100644 release_scripts/localization_scripts/unittestutil.py diff --git a/release_scripts/localization_scripts/testartifacts/.gitignore b/release_scripts/localization_scripts/testartifacts/.gitignore new file mode 100644 index 0000000000..6caf68aff4 --- /dev/null +++ b/release_scripts/localization_scripts/testartifacts/.gitignore @@ -0,0 +1 @@ +output \ No newline at end of file diff --git a/release_scripts/localization_scripts/unittestutil.py b/release_scripts/localization_scripts/unittestutil.py new file mode 100644 index 0000000000..64a090299a --- /dev/null +++ b/release_scripts/localization_scripts/unittestutil.py @@ -0,0 +1 @@ +TEST_OUTPUT_FOLDER = 'testartifacts/output' \ No newline at end of file From e4a0a8490a23557740c24423edc76fa6db1a59a6 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 22 Jul 2020 12:05:55 -0400 Subject: [PATCH 21/42] 6531 pre pull request clean up --- .../autopsy/discovery/DiscoveryTopComponent.java | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/discovery/DiscoveryTopComponent.java b/Core/src/org/sleuthkit/autopsy/discovery/DiscoveryTopComponent.java index c65d5949eb..0d87facce5 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/DiscoveryTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/DiscoveryTopComponent.java @@ -49,17 +49,12 @@ public final class DiscoveryTopComponent extends TopComponent { private static final long serialVersionUID = 1L; private static final String PREFERRED_ID = "Discovery"; // NON-NLS + private static final int ANIMATION_INCREMENT = 30; + private volatile static int resultsAreaSize = 250; private final GroupListPanel groupListPanel; private final DetailsPanel detailsPanel; private final ResultsPanel resultsPanel; private int dividerLocation = -1; - - - - private volatile static int resultsAreaSize = 250; - private volatile static int previousResultsAreaSize = -1; - private static final int ANIMATION_INCREMENT = 30; - private SwingAnimator animator = null; /** @@ -89,11 +84,7 @@ public final class DiscoveryTopComponent extends TopComponent { if (evt.getPropertyName().equalsIgnoreCase(JSplitPane.DIVIDER_LOCATION_PROPERTY)) { //Only change the saved location when it was a manual change by the user and not the animation or the window opening initially if ((animator == null || !animator.isRunning()) && evt.getNewValue() instanceof Integer - && ((int) evt.getNewValue() + 5) < (rightSplitPane.getHeight() - rightSplitPane.getDividerSize()) && (int) evt.getNewValue() != previousResultsAreaSize) { - System.out.println("setting small size" + (int) evt.getNewValue()); - System.out.println("HEIGHT: " + rightSplitPane.getHeight()); - System.out.println("DIVIDER LOC: " + rightSplitPane.getDividerLocation()); - previousResultsAreaSize = (int) evt.getOldValue(); + && ((int) evt.getNewValue() + 5) < (rightSplitPane.getHeight() - rightSplitPane.getDividerSize())) { resultsAreaSize = (int) evt.getNewValue(); } From 9baf804e0ff71098b029d0e525c70bfde4a56c3c Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 22 Jul 2020 15:31:12 -0400 Subject: [PATCH 22/42] testing --- .../localization_scripts/csvutil.py | 43 +------- .../localization_scripts/fileutil.py | 59 ++--------- .../localization_scripts/itemchange.py | 100 +----------------- .../localization_scripts/langpropsutil.py | 1 - .../localization_scripts/propsutil.py | 2 +- .../testartifacts/.gitignore | 1 - .../localization_scripts/unittestutil.py | 1 - 7 files changed, 13 insertions(+), 194 deletions(-) delete mode 100644 release_scripts/localization_scripts/testartifacts/.gitignore delete mode 100644 release_scripts/localization_scripts/unittestutil.py diff --git a/release_scripts/localization_scripts/csvutil.py b/release_scripts/localization_scripts/csvutil.py index eb59b42ad4..daa66f5396 100644 --- a/release_scripts/localization_scripts/csvutil.py +++ b/release_scripts/localization_scripts/csvutil.py @@ -1,12 +1,8 @@ """Provides tools for parsing and writing to a csv file. """ -import codecs -from typing import List, Iterable, Tuple, TypeVar +from typing import List, Iterable, Tuple import csv import os -import unittest -from envutil import get_proj_dir -from unittestutil import TEST_OUTPUT_FOLDER def records_to_csv(output_path: str, rows: Iterable[List[str]]): @@ -53,40 +49,3 @@ def csv_to_records(input_path: str, header_row: bool) -> Tuple[List[List[str]], raise Exception("There was an error parsing csv {path}".format(path=input_path), e) return results, header - - -class CsvUtilTest(unittest.TestCase): - T = TypeVar('T') - - def assert_equal_arr(self, a: List[T], b: List[T]): - self.assertEqual(len(a), len(b), 'arrays are not equal length') - for i in range(0, len(a)): - if isinstance(a[i], list) and isinstance(b[i], list): - self.assert_equal_arr(a[i], b[i]) - else: - self.assertEqual(a[i], b[i], "Items: {0} and {1} at index {2} are not equal.".format(a[i], b[i], i)) - - def test_read_write(self): - data = [['header1', 'header2', 'header3', 'additional header'], - ['data1', 'data2', 'data3'], - ['', 'data2-1', 'data2-2']] - - os.makedirs(os.path.join(get_proj_dir(), TEST_OUTPUT_FOLDER)) - test_path = os.path.join(get_proj_dir(), TEST_OUTPUT_FOLDER, 'test.csv') - records_to_csv(test_path, data) - - byte_inf = min(32, os.path.getsize(test_path)) - with open(test_path, 'rb') as bom_test_file: - raw = bom_test_file.read(byte_inf) - if not raw.startswith(codecs.BOM_UTF8): - self.fail("written csv does not have appropriate BOM") - - read_records_no_header, no_header = csv_to_records(test_path, header_row=False) - self.assert_equal_arr(read_records_no_header, data) - - read_rows, header = csv_to_records(test_path, header_row=True) - self.assert_equal_arr(header, data[0]) - self.assert_equal_arr(read_rows, [data[1], data[2]]) - - - diff --git a/release_scripts/localization_scripts/fileutil.py b/release_scripts/localization_scripts/fileutil.py index 6554b94270..5139812db2 100644 --- a/release_scripts/localization_scripts/fileutil.py +++ b/release_scripts/localization_scripts/fileutil.py @@ -1,6 +1,6 @@ import os -import unittest from typing import Union, Tuple +from pathlib import Path def get_path_pieces(orig_path: str) -> Tuple[str, Union[str, None], Union[str, None]]: @@ -13,11 +13,13 @@ def get_path_pieces(orig_path: str) -> Tuple[str, Union[str, None], Union[str, N """ - potential_parent_dir, orig_file = os.path.split(orig_path) + potential_parent_dir, orig_file = os.path.split(str(Path(orig_path))) filename, file_extension = os.path.splitext(orig_file) + if file_extension.startswith('.'): + file_extension = file_extension[1:] if file_extension is None or len(file_extension) < 1: - return orig_path, None, None + return str(Path(orig_path)), None, None else: return potential_parent_dir, filename, file_extension @@ -35,7 +37,7 @@ def get_new_path(orig_path: str, new_filename: str) -> str: """ parent_dir, filename, ext = get_path_pieces(orig_path) - return os.path.join(parent_dir, new_filename) + return str(Path(parent_dir) / Path(new_filename)) # For use with creating csv filenames for entries that have been omitted. @@ -55,52 +57,7 @@ def get_filename_addition(orig_path: str, filename_addition: str) -> str: """ parent_dir, filename, extension = get_path_pieces(orig_path) if filename is None: - return orig_path + filename_addition + return str(Path(orig_path + filename_addition)) else: ext = '' if extension is None else extension - return os.path.join(parent_dir, '{0}{1}.{2}'.format(filename, filename_addition, ext)) - - -class FileUtilTest(unittest.TestCase): - - @staticmethod - def _joined_paths(pieces: Tuple[str, str, str]) -> str: - return os.path.join(pieces[0], pieces[1] + '.' + pieces[2]) - - PATH_PIECES1 = ('/test/folder', 'filename', 'ext') - PATH_PIECES2 = ('/test.test2/folder.test2', 'filename.test', 'ext') - PATH_PIECES3 = ('/test.test2/folder.test2/folder', None, None) - - def __init__(self): - self.PATH1 = FileUtilTest._joined_paths(self.PATH_PIECES1) - self.PATH2 = FileUtilTest._joined_paths(self.PATH_PIECES2) - self.PATH3 = self.PATH_PIECES3[0] - - self.ALL_ITEMS = [ - (self.PATH_PIECES1, self.PATH1), - (self.PATH_PIECES2, self.PATH2), - (self.PATH_PIECES3, self.PATH3) - ] - - def get_path_pieces_test(self): - for (expected_path, expected_filename, expected_ext), path in self.ALL_ITEMS: - path, filename, ext = get_path_pieces(path) - self.assertEqual(path, expected_path) - self.assertEqual(filename, expected_filename) - self.assertEqual(ext, expected_ext) - - def get_new_path_test(self): - for (expected_path, expected_filename, expected_ext), path in self.ALL_ITEMS: - new_name = "newname.file" - new_path = get_new_path(path, new_name) - self.assertEqual(new_path, os.path.join(expected_path, new_name)) - - def get_filename_addition_test(self): - for (expected_path, expected_filename, expected_ext), path in self.ALL_ITEMS: - addition = "addition" - new_path = get_filename_addition(path, addition) - self.assertEqual( - new_path, os.path.join( - expected_path, - "{file_name}{addition}.{extension}".format( - file_name=expected_filename, addition=addition, extension=expected_ext))) + return str(Path(parent_dir) / Path('{0}{1}.{2}'.format(filename, filename_addition, ext))) diff --git a/release_scripts/localization_scripts/itemchange.py b/release_scripts/localization_scripts/itemchange.py index bfc09b728a..abb81dd3ce 100644 --- a/release_scripts/localization_scripts/itemchange.py +++ b/release_scripts/localization_scripts/itemchange.py @@ -1,5 +1,4 @@ -import unittest -from typing import Iterator, List, Union, Dict +from typing import Iterator, List, Union from propsutil import get_entry_dict from enum import Enum @@ -31,25 +30,13 @@ class ItemChange: self.key = key self.prev_val = prev_val self.cur_val = cur_val - if ItemChange.has_str_content(cur_val) and not ItemChange.has_str_content(prev_val): + if cur_val is not None and prev_val is None: self.type = ChangeType.ADDITION - elif not ItemChange.has_str_content(cur_val) and ItemChange.has_str_content(prev_val): + elif cur_val is None and prev_val is not None: self.type = ChangeType.DELETION else: self.type = ChangeType.CHANGE - @staticmethod - def has_str_content(content: str): - """Determines whether or not the content is empty or None. - - Args: - content (str): The text. - - Returns: - bool: Whether or not it has content. - """ - return content is not None and len(content.strip()) > 0 - @staticmethod def get_headers() -> List[str]: """Returns the csv headers to insert when serializing a list of ItemChange objects to csv. @@ -111,84 +98,3 @@ def get_changed(rel_path: str, a_str: str, b_str: str) -> Iterator[ItemChange]: mapped = map(lambda key: get_item_change( rel_path, key, a_dict.get(key), b_dict.get(key)), all_keys) return filter(lambda entry: entry is not None, mapped) - - -class ItemChangeTest(unittest.TestCase): - @staticmethod - def dict_to_prop_str(dict: Dict[str,str]) -> str: - toret = '' - for key,val in dict.items: - toret += "{key}={value}\n".format(key=key,val=val) - - return toret - - def get_changed_test(self): - deleted_key = 'deleted.property.key' - deleted_val = 'will be deleted' - - change_key = 'change.property.key' - change_val_a = 'original value' - change_val_b = 'new value' - - change_key2 = 'change2.property.key' - change_val2_a = 'original value 2' - change_val2_b = '' - - addition_key = 'addition.property.key' - addition_new_val = 'the added value' - - same_key = 'samevalue.property.key' - same_value = 'the same value' - - same_key2 = 'samevalue2.property.key' - same_value2 = '' - - a_dict = { - deleted_key: deleted_val, - change_key: change_val_a, - change_key2: change_val2_a, - same_key: same_value, - same_key2: same_value2 - } - - b_dict = { - change_key: change_val_b, - change_key2: change_val2_b, - addition_key: addition_new_val, - same_key: same_value, - same_key2: same_value2 - } - - a_str = ItemChangeTest.dict_to_prop_str(a_dict) - b_str = ItemChangeTest.dict_to_prop_str(b_dict) - - rel_path = 'my/rel/path.properties' - - key_to_change = {} - - for item_change in get_changed(rel_path, a_str, b_str): - self.assertEqual(item_change.rel_path, rel_path) - key_to_change[item_change.key] = item_change - - deleted_item = key_to_change[deleted_key] - self.assertEqual(deleted_item.type, ChangeType.DELETION) - self.assertEqual(deleted_item.prev_val, deleted_val) - self.assertEqual(deleted_item.cur_val, None) - - addition_item = key_to_change[addition_key] - self.assertEqual(addition_item.type, ChangeType.ADDITION) - self.assertEqual(addition_item.prev_val, None) - self.assertEqual(deleted_item.cur_val, addition_new_val) - - change_item = key_to_change[change_key] - self.assertEqual(change_item.type, ChangeType.CHANGE) - self.assertEqual(change_item.prev_val, change_val_a) - self.assertEqual(change_item.cur_val, change_val_b) - - change_item2 = key_to_change[change_key2] - self.assertEqual(change_item2.type, ChangeType.CHANGE) - self.assertEqual(change_item2.prev_val, change_val2_a) - self.assertEqual(change_item2.cur_val, change_val2_b) - - self.assertTrue(same_key not in key_to_change) - self.assertTrue(same_key2 not in key_to_change) \ No newline at end of file diff --git a/release_scripts/localization_scripts/langpropsutil.py b/release_scripts/localization_scripts/langpropsutil.py index 1104e14665..841574c9e9 100644 --- a/release_scripts/localization_scripts/langpropsutil.py +++ b/release_scripts/localization_scripts/langpropsutil.py @@ -1,6 +1,5 @@ """Functions handling retrieving and storing when a language was last updated. """ - from typing import Union from envutil import get_proj_dir from propsutil import get_entry_dict_from_path, update_entry_dict diff --git a/release_scripts/localization_scripts/propsutil.py b/release_scripts/localization_scripts/propsutil.py index de94a82271..3de52a7966 100644 --- a/release_scripts/localization_scripts/propsutil.py +++ b/release_scripts/localization_scripts/propsutil.py @@ -1,11 +1,11 @@ """Provides tools for reading from and writing to java properties files. """ - from typing import Dict, Union, IO from jproperties import Properties import os # The default extension for property files in autopsy repo + DEFAULT_PROPS_EXTENSION = 'properties-MERGED' diff --git a/release_scripts/localization_scripts/testartifacts/.gitignore b/release_scripts/localization_scripts/testartifacts/.gitignore deleted file mode 100644 index 6caf68aff4..0000000000 --- a/release_scripts/localization_scripts/testartifacts/.gitignore +++ /dev/null @@ -1 +0,0 @@ -output \ No newline at end of file diff --git a/release_scripts/localization_scripts/unittestutil.py b/release_scripts/localization_scripts/unittestutil.py deleted file mode 100644 index 64a090299a..0000000000 --- a/release_scripts/localization_scripts/unittestutil.py +++ /dev/null @@ -1 +0,0 @@ -TEST_OUTPUT_FOLDER = 'testartifacts/output' \ No newline at end of file From 20c799ddccc8a5f8fea3910dafe089aaaa377261 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 22 Jul 2020 15:32:51 -0400 Subject: [PATCH 23/42] testing --- .../localization_scripts/__init__.py | 0 .../localization_scripts/test/__init__.py | 0 .../test/artifacts/.gitignore | 1 + .../localization_scripts/test/test_csvutil.py | 41 ++++++++ .../test/test_fileutil.py | 52 ++++++++++ .../test/test_itemchange.py | 96 +++++++++++++++++++ .../test/test_propsutil.py | 36 +++++++ .../localization_scripts/test/unittestutil.py | 14 +++ 8 files changed, 240 insertions(+) create mode 100644 release_scripts/localization_scripts/__init__.py create mode 100644 release_scripts/localization_scripts/test/__init__.py create mode 100644 release_scripts/localization_scripts/test/artifacts/.gitignore create mode 100644 release_scripts/localization_scripts/test/test_csvutil.py create mode 100644 release_scripts/localization_scripts/test/test_fileutil.py create mode 100644 release_scripts/localization_scripts/test/test_itemchange.py create mode 100644 release_scripts/localization_scripts/test/test_propsutil.py create mode 100644 release_scripts/localization_scripts/test/unittestutil.py diff --git a/release_scripts/localization_scripts/__init__.py b/release_scripts/localization_scripts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/release_scripts/localization_scripts/test/__init__.py b/release_scripts/localization_scripts/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/release_scripts/localization_scripts/test/artifacts/.gitignore b/release_scripts/localization_scripts/test/artifacts/.gitignore new file mode 100644 index 0000000000..6caf68aff4 --- /dev/null +++ b/release_scripts/localization_scripts/test/artifacts/.gitignore @@ -0,0 +1 @@ +output \ No newline at end of file diff --git a/release_scripts/localization_scripts/test/test_csvutil.py b/release_scripts/localization_scripts/test/test_csvutil.py new file mode 100644 index 0000000000..a5ffd0cb71 --- /dev/null +++ b/release_scripts/localization_scripts/test/test_csvutil.py @@ -0,0 +1,41 @@ +import codecs +import os +import unittest +from typing import TypeVar, List + +from csvutil import records_to_csv, csv_to_records +from test.unittestutil import get_output_path + + +class CsvUtilTest(unittest.TestCase): + T = TypeVar('T') + + def assert_equal_arr(self, a: List[T], b: List[T]): + self.assertEqual(len(a), len(b), 'arrays are not equal length') + for i in range(0, len(a)): + if isinstance(a[i], list) and isinstance(b[i], list): + self.assert_equal_arr(a[i], b[i]) + else: + self.assertEqual(a[i], b[i], "Items: {0} and {1} at index {2} are not equal.".format(a[i], b[i], i)) + + def test_read_write(self): + data = [['header1', 'header2', 'header3', 'additional header'], + ['data1', 'data2', 'data3'], + ['', 'data2-1', 'data2-2']] + + os.makedirs(get_output_path(), exist_ok=True) + test_path = get_output_path('test.csv') + records_to_csv(test_path, data) + + byte_inf = min(32, os.path.getsize(test_path)) + with open(test_path, 'rb') as bom_test_file: + raw = bom_test_file.read(byte_inf) + if not raw.startswith(codecs.BOM_UTF8): + self.fail("written csv does not have appropriate BOM") + + read_records_no_header, no_header = csv_to_records(test_path, header_row=False) + self.assert_equal_arr(read_records_no_header, data) + + read_rows, header = csv_to_records(test_path, header_row=True) + self.assert_equal_arr(header, data[0]) + self.assert_equal_arr(read_rows, [data[1], data[2]]) diff --git a/release_scripts/localization_scripts/test/test_fileutil.py b/release_scripts/localization_scripts/test/test_fileutil.py new file mode 100644 index 0000000000..290396eba7 --- /dev/null +++ b/release_scripts/localization_scripts/test/test_fileutil.py @@ -0,0 +1,52 @@ +import os +import unittest +from typing import Tuple +from pathlib import Path +from fileutil import get_path_pieces, get_new_path, get_filename_addition + + +def joined_paths(pieces: Tuple[str, str, str]) -> str: + return os.path.join(pieces[0], pieces[1] + '.' + pieces[2]) + + +PATH_PIECES1 = ('/test/folder', 'filename', 'ext') +PATH_PIECES2 = ('/test.test2/folder.test2', 'filename.test', 'ext') +PATH_PIECES3 = ('/test.test2/folder.test2/folder', None, None) + +PATH1 = joined_paths(PATH_PIECES1) +PATH2 = joined_paths(PATH_PIECES2) +PATH3 = PATH_PIECES3[0] + +ALL_ITEMS = [ + (PATH_PIECES1, PATH1), + (PATH_PIECES2, PATH2), + (PATH_PIECES3, PATH3) +] + + +class FileUtilTest(unittest.TestCase): + def test_get_path_pieces(self): + for (expected_path, expected_filename, expected_ext), path in ALL_ITEMS: + path, filename, ext = get_path_pieces(path) + self.assertEqual(path, str(Path(expected_path))) + self.assertEqual(filename, expected_filename) + self.assertEqual(ext, expected_ext) + + def test_get_new_path(self): + for (expected_path, expected_filename, expected_ext), path in ALL_ITEMS: + new_name = "newname.file" + new_path = get_new_path(path, new_name) + self.assertEqual(new_path, str(Path(expected_path) / Path(new_name))) + + def test_get_filename_addition(self): + for (expected_path, expected_filename, expected_ext), path in ALL_ITEMS: + addition = "addition" + new_path = get_filename_addition(path, addition) + if expected_filename is None or expected_ext is None: + expected_file_path = Path(expected_path + addition) + else: + expected_file_path = Path(expected_path) / Path("{file_name}{addition}.{extension}".format( + file_name=expected_filename, addition=addition, extension=expected_ext)) + + self.assertEqual( + new_path, str(expected_file_path)) diff --git a/release_scripts/localization_scripts/test/test_itemchange.py b/release_scripts/localization_scripts/test/test_itemchange.py new file mode 100644 index 0000000000..91b7846a11 --- /dev/null +++ b/release_scripts/localization_scripts/test/test_itemchange.py @@ -0,0 +1,96 @@ +import unittest +from typing import Dict + +from itemchange import get_changed, ChangeType + + +def dict_to_prop_str(this_dict: Dict[str, str]) -> str: + toret = '' + for key, val in this_dict.items(): + toret += "{key}={value}\n".format(key=key, value=val) + + return toret + + +class ItemChangeTest(unittest.TestCase): + def test_get_changed(self): + deleted_key = 'deleted.property.key' + deleted_val = 'will be deleted' + + change_key = 'change.property.key' + change_val_a = 'original value' + change_val_b = 'new value' + + change_key2 = 'change2.property.key' + change_val2_a = 'original value 2' + change_val2_b = '' + + change_key3 = 'change3.property.key' + change_val3_a = '' + change_val3_b = 'cur value 3' + + addition_key = 'addition.property.key' + addition_new_val = 'the added value' + + same_key = 'samevalue.property.key' + same_value = 'the same value' + + same_key2 = 'samevalue2.property.key' + same_value2 = '' + + a_dict = { + deleted_key: deleted_val, + change_key: change_val_a, + change_key2: change_val2_a, + change_key3: change_val3_a, + same_key: same_value, + same_key2: same_value2 + } + + b_dict = { + change_key: change_val_b, + change_key2: change_val2_b, + change_key3: change_val3_b, + addition_key: addition_new_val, + same_key: same_value, + same_key2: same_value2 + } + + a_str = dict_to_prop_str(a_dict) + b_str = dict_to_prop_str(b_dict) + + rel_path = 'my/rel/path.properties' + + key_to_change = {} + + for item_change in get_changed(rel_path, a_str, b_str): + self.assertEqual(item_change.rel_path, rel_path) + key_to_change[item_change.key] = item_change + + deleted_item = key_to_change[deleted_key] + self.assertEqual(deleted_item.type, ChangeType.DELETION) + self.assertEqual(deleted_item.prev_val, deleted_val) + self.assertEqual(deleted_item.cur_val, None) + + addition_item = key_to_change[addition_key] + self.assertEqual(addition_item.type, ChangeType.ADDITION) + self.assertEqual(addition_item.prev_val, None) + self.assertEqual(addition_item.cur_val, addition_new_val) + + change_item = key_to_change[change_key] + self.assertEqual(change_item.type, ChangeType.CHANGE) + self.assertEqual(change_item.prev_val, change_val_a) + self.assertEqual(change_item.cur_val, change_val_b) + + change_item2 = key_to_change[change_key2] + self.assertEqual(change_item2.type, ChangeType.CHANGE) + self.assertEqual(change_item2.prev_val, change_val2_a) + self.assertEqual(change_item2.cur_val, change_val2_b) + + change_item3 = key_to_change[change_key3] + self.assertEqual(change_item3.type, ChangeType.CHANGE) + self.assertEqual(change_item3.prev_val, change_val3_a) + self.assertEqual(change_item3.cur_val, change_val3_b) + + self.assertTrue(same_key not in key_to_change) + self.assertTrue(same_key2 not in key_to_change) diff --git a/release_scripts/localization_scripts/test/test_propsutil.py b/release_scripts/localization_scripts/test/test_propsutil.py new file mode 100644 index 0000000000..f69129399a --- /dev/null +++ b/release_scripts/localization_scripts/test/test_propsutil.py @@ -0,0 +1,36 @@ +import os +import unittest + +from propsutil import set_entry_dict, get_entry_dict_from_path, update_entry_dict +from test.unittestutil import get_output_path + + +class PropsUtilTest(unittest.TestCase): + def test_update_entry_dict(self): + orig_key = 'orig_key' + orig_val = 'orig_val 片仮名 ' + to_be_altered_key = 'tobealteredkey' + first_val = 'not yet altered sábado' + second_val = 'altered Stöcke' + + orig_props = { + orig_key: orig_val, + to_be_altered_key: first_val + } + + update_props = { + to_be_altered_key: second_val + } + + os.makedirs(get_output_path(), exist_ok=True) + test_path = get_output_path('test.props') + set_entry_dict(orig_props, test_path) + + orig_read_props = get_entry_dict_from_path(test_path) + self.assertEqual(orig_read_props[orig_key], orig_val) + self.assertEqual(orig_read_props[to_be_altered_key], first_val) + + update_entry_dict(update_props, test_path) + updated_read_props = get_entry_dict_from_path(test_path) + self.assertEqual(updated_read_props[orig_key], orig_val) + self.assertEqual(updated_read_props[to_be_altered_key], second_val) diff --git a/release_scripts/localization_scripts/test/unittestutil.py b/release_scripts/localization_scripts/test/unittestutil.py new file mode 100644 index 0000000000..19face5610 --- /dev/null +++ b/release_scripts/localization_scripts/test/unittestutil.py @@ -0,0 +1,14 @@ +import os +from typing import Union + +from envutil import get_proj_dir + +TEST_ARTIFACT_FOLDER = 'artifacts' +TEST_OUTPUT_FOLDER = 'output' + + +def get_output_path(filename: Union[str, None] = None) -> str: + if filename is None: + return os.path.join(get_proj_dir(__file__), TEST_ARTIFACT_FOLDER, TEST_OUTPUT_FOLDER) + else: + return os.path.join(get_proj_dir(__file__), TEST_ARTIFACT_FOLDER, TEST_OUTPUT_FOLDER, filename) From 1d443f6cc86b88c44ad7dc334415169ada0ff762 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 22 Jul 2020 15:43:38 -0400 Subject: [PATCH 24/42] testing --- release_scripts/localization_scripts/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/release_scripts/localization_scripts/README.md b/release_scripts/localization_scripts/README.md index 0b738f8f38..61c10c334a 100644 --- a/release_scripts/localization_scripts/README.md +++ b/release_scripts/localization_scripts/README.md @@ -14,4 +14,7 @@ All of these scripts provide more details on usage by calling the script with `- 3. Call `python3 updatepropsscript.py -l ` to update properties files based on the newly generated csv file. The csv file should be formatted such that the columns are bundle relative path, property files key, translated value and commit id for the latest commit id for which these changes represent. The commit id only needs to be in the header row. ## Localization Generation for the First Time -First-time updates should follow a similar procedure except that instead of calling `diffscript.py`, call `python3 allbundlesscript ` to generate a csv file with relative paths of bundle files, property file keys, property file values. \ No newline at end of file +First-time updates should follow a similar procedure except that instead of calling `diffscript.py`, call `python3 allbundlesscript ` to generate a csv file with relative paths of bundle files, property file keys, property file values. + +##Unit Tests +Unit tests can be run from this directory using `python3 -m unittest` \ No newline at end of file From 35e4be5cd9a836ae828951bc6fd46c1ce327e9f1 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 22 Jul 2020 16:10:40 -0400 Subject: [PATCH 25/42] some debugging --- release_scripts/localization_scripts/diffscript.py | 7 ++++--- release_scripts/localization_scripts/gitutil.py | 6 +++--- .../localization_scripts/lastupdated.properties | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index df805d7236..4f207f3f9f 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -65,7 +65,7 @@ def main(): help='The commit for current release.') parser.add_argument('-nc', '--no-commits', dest='no_commits', action='store_true', default=False, required=False, help="Suppresses adding commits to the generated csv header.") - parser.add_argument('-l', '--language', dest='language', type=str, default='HEAD', required=False, + parser.add_argument('-l', '--language', dest='language', type=str, default=None, required=False, help='Specify the language in order to determine the first commit to use (i.e. \'ja\' for ' 'Japanese. This flag overrides the first-commit flag.') parser.add_argument('-vr', '--value-regex', dest='value_regex', type=str, default=None, required=False, @@ -81,8 +81,9 @@ def main(): output_path = args.output_path commit_1_id = args.commit_1_id value_regex = args.value_regex - if args.language is not None: - commit_1_id = get_commit_for_language(args.language) + lang = args.language + if lang is not None: + commit_1_id = get_commit_for_language(lang) if commit_1_id is None: print('Either the first commit or language flag need to be specified. If specified, the language file, ' + diff --git a/release_scripts/localization_scripts/gitutil.py b/release_scripts/localization_scripts/gitutil.py index 333bee1440..43c20e2ce0 100644 --- a/release_scripts/localization_scripts/gitutil.py +++ b/release_scripts/localization_scripts/gitutil.py @@ -100,7 +100,7 @@ def get_commit_id(repo_path: str, commit_id: str) -> str: The hash for the commit in the repo. """ repo = Repo(repo_path, search_parent_directories=True) - commit = repo.commit(commit_id) + commit = repo.commit(commit_id.strip()) return str(commit.hexsha) @@ -118,7 +118,7 @@ def get_property_files_diff(repo_path: str, commit_1_id: str, commit_2_id: str, All found item changes in values of keys between the property files. """ - diffs = get_diff(repo_path, commit_1_id, commit_2_id) + diffs = get_diff(repo_path, commit_1_id.strip(), commit_2_id.strip()) for diff in diffs: rel_path = get_rel_path(diff) if rel_path is None or not rel_path.endswith('.' + property_file_extension): @@ -160,7 +160,7 @@ def get_property_file_entries(repo_path: str, at_commit: str = 'HEAD', """ repo = Repo(repo_path, search_parent_directories=True) - commit = repo.commit(at_commit) + commit = repo.commit(at_commit.strip()) for item in list_paths(commit.tree): path, blob = item if path.endswith(property_file_extension): diff --git a/release_scripts/localization_scripts/lastupdated.properties b/release_scripts/localization_scripts/lastupdated.properties index e69de29bb2..8a337511e7 100644 --- a/release_scripts/localization_scripts/lastupdated.properties +++ b/release_scripts/localization_scripts/lastupdated.properties @@ -0,0 +1 @@ +# in format of bundles..lastupdated= \ No newline at end of file From 325d875fcafb9fb89afbb0d3122f172c496789d8 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 23 Jul 2020 07:25:43 -0400 Subject: [PATCH 26/42] removed value regex flags --- .../localization_scripts/allbundlesscript.py | 11 ++--------- release_scripts/localization_scripts/diffscript.py | 12 ++---------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py index 71146ffc91..38426d824a 100644 --- a/release_scripts/localization_scripts/allbundlesscript.py +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -15,7 +15,7 @@ import re import argparse -def write_items_to_csv(repo_path: str, output_path: str, show_commit: bool, value_regex: Union[str, None]): +def write_items_to_csv(repo_path: str, output_path: str, show_commit: bool, value_regex: Union[str, None] = None): """Determines the contents of '.properties-MERGED' files and writes to a csv file. Args: @@ -56,20 +56,13 @@ def main(): help='The path to the repo. If not specified, path of script is used.') parser.add_argument('-nc', '--no_commit', dest='no_commit', action='store_true', default=False, required=False, help="Suppresses adding commits to the generated csv header.") - parser.add_argument('-vr', '--value-regex', dest='value_regex', type=str, default=None, required=False, - help='Specify the regex for the property value where a regex match against the property value ' - 'will display the key value pair in csv output (i.e. \'[a-zA-Z]\' or \'\\S\' for removing ' - 'just whitespace items). If this option is not specified, all key value pairs will be ' - 'accepted. If this option is specified, another csv file will be created in the same ' - 'directory as the original output_path with \'' + OMITTED_ADDITION + ' appended.') args = parser.parse_args() repo_path = args.repo_path if args.repo_path is not None else get_git_root(get_proj_dir()) output_path = args.output_path show_commit = not args.no_commit - value_regex = args.value_regex - write_items_to_csv(repo_path, output_path, show_commit, value_regex) + write_items_to_csv(repo_path, output_path, show_commit) sys.exit(0) diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index 4f207f3f9f..259191292e 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -15,7 +15,7 @@ from langpropsutil import get_commit_for_language, LANG_FILENAME def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit_2_id: str, show_commits: bool, - value_regex: Union[str, None]): + value_regex: Union[str, None] = None): """Determines the changes made in '.properties-MERGED' files from one commit to another commit. Args: @@ -68,19 +68,11 @@ def main(): parser.add_argument('-l', '--language', dest='language', type=str, default=None, required=False, help='Specify the language in order to determine the first commit to use (i.e. \'ja\' for ' 'Japanese. This flag overrides the first-commit flag.') - parser.add_argument('-vr', '--value-regex', dest='value_regex', type=str, default=None, required=False, - help='Specify the regex for the property value where a regex match against the property value ' - 'will display the key value pair in csv output (i.e. \'[a-zA-Z]\' or \'\\S\' for removing ' - 'just whitespace items). If this option is not specified, all key value pairs will be ' - 'accepted. If this option is specified, entries marked as \'DELETION\' will also be ' - 'omitted another csv file will be created in the same directory as the original ' - 'output_path with \'' + OMITTED_ADDITION + ' appended.') args = parser.parse_args() repo_path = args.repo_path if args.repo_path is not None else get_git_root(get_proj_dir()) output_path = args.output_path commit_1_id = args.commit_1_id - value_regex = args.value_regex lang = args.language if lang is not None: commit_1_id = get_commit_for_language(lang) @@ -94,7 +86,7 @@ def main(): commit_2_id = args.commit_2_id show_commits = not args.no_commits - write_diff_to_csv(repo_path, output_path, commit_1_id, commit_2_id, show_commits, value_regex) + write_diff_to_csv(repo_path, output_path, commit_1_id, commit_2_id, show_commits) sys.exit(0) From eedef0d14438f9f24315b5749b66ef95e3bec35d Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 23 Jul 2020 07:36:45 -0400 Subject: [PATCH 27/42] fix for change type output --- release_scripts/localization_scripts/itemchange.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/release_scripts/localization_scripts/itemchange.py b/release_scripts/localization_scripts/itemchange.py index abb81dd3ce..27448cb529 100644 --- a/release_scripts/localization_scripts/itemchange.py +++ b/release_scripts/localization_scripts/itemchange.py @@ -9,6 +9,9 @@ class ChangeType(Enum): DELETION = 'DELETION' CHANGE = 'CHANGE' + def __str__(self): + return str(self.value) + class ItemChange: rel_path: str From daa5900cd47f78dfb4c477f0cba0b50cdbe276c0 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 23 Jul 2020 08:16:00 -0400 Subject: [PATCH 28/42] small grammar fix --- release_scripts/localization_scripts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release_scripts/localization_scripts/README.md b/release_scripts/localization_scripts/README.md index 61c10c334a..408e868cf6 100644 --- a/release_scripts/localization_scripts/README.md +++ b/release_scripts/localization_scripts/README.md @@ -17,4 +17,4 @@ All of these scripts provide more details on usage by calling the script with `- First-time updates should follow a similar procedure except that instead of calling `diffscript.py`, call `python3 allbundlesscript ` to generate a csv file with relative paths of bundle files, property file keys, property file values. ##Unit Tests -Unit tests can be run from this directory using `python3 -m unittest` \ No newline at end of file +Unit tests can be run from this directory using `python3 -m unittest`. \ No newline at end of file From d89d7af2e3bd12ae6690b635ed35cc733744c2a2 Mon Sep 17 00:00:00 2001 From: apriestman Date: Thu, 23 Jul 2020 15:45:43 -0400 Subject: [PATCH 29/42] Added more modules to the list in the quick start guide. --- docs/doxygen-user/quick_start_guide.dox | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/doxygen-user/quick_start_guide.dox b/docs/doxygen-user/quick_start_guide.dox index 0a279244cf..19b3a44c1d 100644 --- a/docs/doxygen-user/quick_start_guide.dox +++ b/docs/doxygen-user/quick_start_guide.dox @@ -22,8 +22,7 @@ The next step is to add an input data source to the case. The Add Data S - For local disk, select one of the detected disks. Autopsy will add the current view of the disk to the case (i.e. snapshot of the meta-data). However, the individual file content (not meta-data) does get updated with the changes made to the disk. You can optionally create a copy of all data read from the local disk to a VHD file, which can be useful for triage situations. Note, you may need run Autopsy as an Administrator to detect all disks. - For logical files (a single file or folder of files), use the "Add" button to add one or more files or folders on your system to the case. Folders will be recursively added to the case. - -After supplying the needed data, Autopsy will quickly review the data sources and add minimal metadata to the case databases so that it can schedule the files for analysis. While it is doing that, it will prompt you to configure the Ingest Modules. +Next it will prompt you to configure the Ingest Modules. \subsection s1c Ingest Modules @@ -35,18 +34,21 @@ The standard ingest modules included with Autopsy are: - \subpage recent_activity_page extracts user activity as saved by web browsers and the OS. Also runs Regripper on the registry hive. - \subpage hash_db_page uses hash sets to ignore known files from the NIST NSRL and flag known bad files. Use the "Advanced" button to add and configure the hash sets to use during this process. You will get updates on known bad file hits as the ingest occurs. You can later add hash sets via the Tools -> Options menu in the main UI. You can download an index of the NIST NSRL from http://sourceforge.net/projects/autopsy/files/NSRL/ - \subpage file_type_identification_page determines file types based on signatures and reports them based on MIME type. It stores the results in the Blackboard and many modules depend on this. It uses the Tika open source library. You can define your own custom file types in Tools, Options, File Types. +- \subpage extension_mismatch_detector_page uses the results from the File Type Identification and flags files that have an extension not traditionally associated with the file's detected type. Ignores 'known' (NSRL) files. You can customize the MIME types and file extensions per MIME type in Tools, Options, File Extension Mismatch. - \subpage embedded_file_extractor_page opens ZIP, RAR, other archive formats, Doc, Docx, PPT, PPTX, XLS, and XLSX and sends the derived files from those files back through the ingest pipeline for analysis. - \subpage EXIF_parser_page extracts EXIF information from JPEG files and posts the results into the tree in the main UI. - \subpage keyword_search_page uses keyword lists to identify files with specific words in them. You can select the keyword lists to search for automatically and you can create new lists using the "Advanced" button. Note that with keyword search, you can always conduct searches after ingest has finished. The keyword lists that you select during ingest will be searched for at periodic intervals and you will get the results in real-time. You do not need to wait for all files to be indexed before performing a keyword search, however you will only get results from files that have already been indexed when you perform your search. - \subpage email_parser_page identifies Thunderbird MBOX files and PST format files based on file signatures, extracting the e-mails from them, adding the results to the Blackboard. -- \subpage extension_mismatch_detector_page uses the results from the File Type Identification and flags files that have an extension not traditionally associated with the file's detected type. Ignores 'known' (NSRL) files. You can customize the MIME types and file extensions per MIME type in Tools, Options, File Extension Mismatch. -- \subpage data_source_integrity_page computes a checksum on E01 files and compares with the E01 file's internal checksum to ensure they match. -- \subpage android_analyzer_page allows you to parse common items from Android devices. Places artifacts into the BlackBoard. -- \subpage interesting_files_identifier_page searches for files and directories based on user-specified rules in Tools, Options, Interesting Files. It works as a "File Alerting Module". It generates messages in the inbox when specified files are found. -- \subpage photorec_carver_page carves files from unallocated space and sends them through the file processing chain. -- \subpage cr_ingest_module adds file hashes and other extracted properties to a central repository for future correlation and to flag previously notable files. - \subpage encryption_page looks for encrypted files. +- \subpage interesting_files_identifier_page searches for files and directories based on user-specified rules in Tools, Options, Interesting Files. It works as a "File Alerting Module". It generates messages in the inbox when specified files are found. +- \subpage cr_ingest_module adds file hashes and other extracted properties to a central repository for future correlation and to flag previously notable files. +- \subpage photorec_carver_page carves files from unallocated space and sends them through the file processing chain. - \subpage vm_extractor_page extracts data from virtual machine files +- \subpage data_source_integrity_page computes a checksum on E01 files and compares with the E01 file's internal checksum to ensure they match. +- \subpage drone_page extracts data from drone files. +- \subpage plaso_page uses Plaso to create \ref timeline_page "timeline" events. +- \subpage android_analyzer_page allows you to parse common items from Android devices. Places artifacts into the BlackBoard. +- \subpage gpx_page extracts geolocation data from .gpx files. When you select a module, you will have the option to change its settings. For example, you can configure which keyword search lists to use during ingest and which hash sets to use. Refer to the individual module help for details on configuring each module. From e38ff77d70cca2beeacc37fc2b222b4b04626c8a Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Fri, 24 Jul 2020 12:03:20 -0400 Subject: [PATCH 30/42] Fixed cvt and geolocation multiuser issues --- .../autopsy/communications/CVTFilterRefresher.java | 2 ++ .../org/sleuthkit/autopsy/geolocation/GeoFilterPanel.java | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/communications/CVTFilterRefresher.java b/Core/src/org/sleuthkit/autopsy/communications/CVTFilterRefresher.java index 1ba9d6c81e..7b2a2cf70e 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/CVTFilterRefresher.java +++ b/Core/src/org/sleuthkit/autopsy/communications/CVTFilterRefresher.java @@ -65,8 +65,10 @@ abstract class CVTFilterRefresher implements RefreshThrottler.Refresher { try (SleuthkitCase.CaseDbQuery dbQuery = skCase.executeQuery("SELECT MAX(date_time) as end, MIN(date_time) as start from account_relationships")) { // ResultSet is closed by CasDBQuery ResultSet rs = dbQuery.getResultSet(); + startTime = rs.getInt("start"); // NON-NLS endTime = rs.getInt("end"); // NON-NLS + } // Get the devices with CVT artifacts List deviceObjIds = new ArrayList<>(); diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.java b/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.java index 2aa820b12f..c5d945dd31 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/GeoFilterPanel.java @@ -501,9 +501,9 @@ class GeoFilterPanel extends javax.swing.JPanel { DataSource dataSource, BlackboardArtifact.ARTIFACT_TYPE artifactType) throws TskCoreException { long count = 0; String queryStr - = "SELECT count(DISTINCT artifact_id) AS count FROM" + = "SELECT count(DISTINCT artIds) AS count FROM" + " (" - + " SELECT * FROM blackboard_artifacts as arts" + + " SELECT arts.artifact_id as artIds, * FROM blackboard_artifacts as arts" + " INNER JOIN blackboard_attributes as attrs" + " ON attrs.artifact_id = arts.artifact_id" + " WHERE arts.artifact_type_id = " + artifactType.getTypeID() @@ -516,7 +516,7 @@ class GeoFilterPanel extends javax.swing.JPanel { + " or attrs.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_TRACKPOINTS.getTypeID() + " or attrs.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_WAYPOINTS.getTypeID() + " )" - + " )"; + + " ) as innerTable"; try (SleuthkitCase.CaseDbQuery queryResult = sleuthkitCase.executeQuery(queryStr); ResultSet resultSet = queryResult.getResultSet()) { if (resultSet.next()) { From 559c5ccc1dbe289ed77bb19c17302d45ba5ed844 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Fri, 24 Jul 2020 13:55:50 -0400 Subject: [PATCH 31/42] Added missing line for cvt fix --- .../sleuthkit/autopsy/communications/CVTFilterRefresher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/communications/CVTFilterRefresher.java b/Core/src/org/sleuthkit/autopsy/communications/CVTFilterRefresher.java index 7b2a2cf70e..b5560ab825 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/CVTFilterRefresher.java +++ b/Core/src/org/sleuthkit/autopsy/communications/CVTFilterRefresher.java @@ -65,7 +65,7 @@ abstract class CVTFilterRefresher implements RefreshThrottler.Refresher { try (SleuthkitCase.CaseDbQuery dbQuery = skCase.executeQuery("SELECT MAX(date_time) as end, MIN(date_time) as start from account_relationships")) { // ResultSet is closed by CasDBQuery ResultSet rs = dbQuery.getResultSet(); - + rs.next(); startTime = rs.getInt("start"); // NON-NLS endTime = rs.getInt("end"); // NON-NLS From 2b0384eaa48dc799f3aed6081743273737149db7 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 24 Jul 2020 18:15:57 -0400 Subject: [PATCH 32/42] 6648 de-select instance when closing discovery to clear content viewer --- .../org/sleuthkit/autopsy/discovery/DiscoveryTopComponent.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/discovery/DiscoveryTopComponent.java b/Core/src/org/sleuthkit/autopsy/discovery/DiscoveryTopComponent.java index 0d87facce5..5e98e05476 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/DiscoveryTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/DiscoveryTopComponent.java @@ -143,9 +143,11 @@ public final class DiscoveryTopComponent extends TopComponent { @Override protected void componentClosed() { DiscoveryDialog.getDiscoveryDialogInstance().cancelSearch(); + DiscoveryEventUtils.getDiscoveryEventBus().post(new DiscoveryEventUtils.ClearInstanceSelectionEvent()); DiscoveryEventUtils.getDiscoveryEventBus().unregister(this); DiscoveryEventUtils.getDiscoveryEventBus().unregister(groupListPanel); DiscoveryEventUtils.getDiscoveryEventBus().unregister(resultsPanel); + DiscoveryEventUtils.getDiscoveryEventBus().unregister(detailsPanel); super.componentClosed(); } From ecabe0c0a6b3a449270e5605bac9739cd924b4cb Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 24 Jul 2020 18:19:30 -0400 Subject: [PATCH 33/42] 6648 remove extra line --- .../org/sleuthkit/autopsy/discovery/DiscoveryTopComponent.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/discovery/DiscoveryTopComponent.java b/Core/src/org/sleuthkit/autopsy/discovery/DiscoveryTopComponent.java index 5e98e05476..633bf5c612 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/DiscoveryTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/DiscoveryTopComponent.java @@ -147,7 +147,6 @@ public final class DiscoveryTopComponent extends TopComponent { DiscoveryEventUtils.getDiscoveryEventBus().unregister(this); DiscoveryEventUtils.getDiscoveryEventBus().unregister(groupListPanel); DiscoveryEventUtils.getDiscoveryEventBus().unregister(resultsPanel); - DiscoveryEventUtils.getDiscoveryEventBus().unregister(detailsPanel); super.componentClosed(); } From 44765d194055e7856d8c24a3d20f1683fda1075e Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 28 Jul 2020 10:38:50 -0400 Subject: [PATCH 34/42] Reduced WARNING logging for KWHs in the CaseUcoReportModule --- .../modules/caseuco/CaseUcoReportModule.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java b/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java index 1523005d92..acb0669738 100755 --- a/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java +++ b/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java @@ -201,21 +201,29 @@ public final class CaseUcoReportModule implements GeneralReportModule { Set dataSourceIds = dataSources.stream() .map((datasource) -> datasource.getId()) .collect(Collectors.toSet()); + + logger.log(Level.INFO, "Writing all artifacts to the CASE-UCO report. " + + "Keyword hits will be skipped as they can't be represented" + + " in CASE format."); // Write all standard artifacts that are contained within the // selected data sources. for (ARTIFACT_TYPE artType : currentCase.getSleuthkitCase().getBlackboardArtifactTypesInUse()) { + if(artType.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT)) { + // Keyword hits cannot be represented in CASE. + continue; + } + for (BlackboardArtifact artifact : currentCase.getSleuthkitCase().getBlackboardArtifacts(artType)) { if (dataSourceIds.contains(artifact.getDataSource().getId())) { - try { for (JsonElement element : exporter.exportBlackboardArtifact(artifact)) { gson.toJson(element, reportWriter); } } catch (ContentNotExportableException | BlackboardJsonAttrUtil.InvalidJsonException ex) { - logger.log(Level.WARNING, String.format("Unable to export blackboard artifact (id: %d) to CASE/UCO. " + logger.log(Level.WARNING, String.format("Unable to export blackboard artifact (id: %d, type: %d) to CASE/UCO. " + "The artifact type is either not supported or the artifact instance does not have any " - + "exportable attributes.", artifact.getId())); + + "exportable attributes.", artifact.getId(), artType.getTypeID())); } } } From 42df6a971f6d6cb2cba8424ef059080b44762384 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 28 Jul 2020 10:55:03 -0400 Subject: [PATCH 35/42] Demoted the warning to an info --- .../autopsy/report/modules/caseuco/CaseUcoReportModule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java b/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java index acb0669738..d6410b7983 100755 --- a/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java +++ b/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java @@ -221,7 +221,7 @@ public final class CaseUcoReportModule implements GeneralReportModule { gson.toJson(element, reportWriter); } } catch (ContentNotExportableException | BlackboardJsonAttrUtil.InvalidJsonException ex) { - logger.log(Level.WARNING, String.format("Unable to export blackboard artifact (id: %d, type: %d) to CASE/UCO. " + logger.log(Level.INFO, String.format("Unable to export blackboard artifact (id: %d, type: %d) to CASE/UCO. " + "The artifact type is either not supported or the artifact instance does not have any " + "exportable attributes.", artifact.getId(), artType.getTypeID())); } From f56ce617eb2fb97297c67a1ae058fe437f68309a Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 28 Jul 2020 10:58:28 -0400 Subject: [PATCH 36/42] Made error logging more granular --- .../autopsy/report/modules/caseuco/CaseUcoReportModule.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java b/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java index d6410b7983..2fc77335be 100755 --- a/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java +++ b/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java @@ -220,10 +220,13 @@ public final class CaseUcoReportModule implements GeneralReportModule { for (JsonElement element : exporter.exportBlackboardArtifact(artifact)) { gson.toJson(element, reportWriter); } - } catch (ContentNotExportableException | BlackboardJsonAttrUtil.InvalidJsonException ex) { + } catch (ContentNotExportableException ex) { logger.log(Level.INFO, String.format("Unable to export blackboard artifact (id: %d, type: %d) to CASE/UCO. " + "The artifact type is either not supported or the artifact instance does not have any " + "exportable attributes.", artifact.getId(), artType.getTypeID())); + } catch (BlackboardJsonAttrUtil.InvalidJsonException ex) { + logger.log(Level.WARNING, String.format("Artifact instance (id: %d, type: %d) contained a " + + "malformed json attribute.", artifact.getId(), artType.getTypeID()), ex); } } } From 97574e81649d3c23bc43ef2cb8160459ddeb2f3e Mon Sep 17 00:00:00 2001 From: apriestman Date: Tue, 28 Jul 2020 12:05:47 -0400 Subject: [PATCH 37/42] Don't print stack trace for invalid ID --- .../centralrepository/datamodel/CentralRepoAccount.java | 2 +- .../centralrepository/datamodel/CorrelationAttributeUtil.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoAccount.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoAccount.java index 90e61fbd0d..88c74a4fe8 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoAccount.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoAccount.java @@ -305,7 +305,7 @@ public final class CentralRepoAccount { normalizedAccountIdentifier = accountIdentifier.toLowerCase().trim(); } } catch (CorrelationAttributeNormalizationException ex) { - throw new InvalidAccountIDException("Failed to normalize the account idenitier.", ex); + throw new InvalidAccountIDException("Failed to normalize the account idenitier " + accountIdentifier, ex); } return normalizedAccountIdentifier; } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java index 71efaa52b8..9af6fcde3b 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java @@ -185,11 +185,11 @@ public class CorrelationAttributeUtil { } } } catch (CorrelationAttributeNormalizationException ex) { - logger.log(Level.SEVERE, String.format("Error normalizing correlation attribute (%s)", artifact), ex); // NON-NLS + logger.log(Level.WARNING, String.format("Error normalizing correlation attribute (%s)", artifact), ex); // NON-NLS return correlationAttrs; } catch (InvalidAccountIDException ex) { - logger.log(Level.SEVERE, String.format("Invalid account identifier (%s)", artifact), ex); // NON-NLS + logger.log(Level.WARNING, String.format("Invalid account identifier (artifactID: %d)", artifact.getId())); // NON-NLS return correlationAttrs; } catch (CentralRepoException ex) { From 0bcf6aca329e93ae04692f3c23a322db758ef2da Mon Sep 17 00:00:00 2001 From: esaunders Date: Tue, 28 Jul 2020 14:22:06 -0400 Subject: [PATCH 38/42] TSK now requires NuGet to be installed and packages to be restored. --- appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 56582bbe43..925d983670 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,6 +23,7 @@ environment: PYTHON: "C:\\Python36-x64" install: + - ps: choco install nuget.commandline - ps: choco install ant --ignore-dependencies - git clone https://github.com/sleuthkit/sleuthkit - ps: $env:Path="C:\Program Files\Java\jdk1.8.0\bin;$($env:Path);C:\ProgramData\chocolatey\lib\ant" @@ -36,6 +37,7 @@ services: build_script: - cd %TSK_HOME% + - nuget restore win32\libtsk -PackagesDirectory win32\packages - python setupDevRepos.py - python win32\updateAndBuildAll.py -m - ps: pushd bindings/java From 443fca9b5a669f637937d05b5d529f8a5c790b1a Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 28 Jul 2020 16:04:45 -0400 Subject: [PATCH 39/42] last commit for Japanese set to autopsy-4.15.0 --- release_scripts/localization_scripts/lastupdated.properties | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release_scripts/localization_scripts/lastupdated.properties b/release_scripts/localization_scripts/lastupdated.properties index 8a337511e7..db7e961472 100644 --- a/release_scripts/localization_scripts/lastupdated.properties +++ b/release_scripts/localization_scripts/lastupdated.properties @@ -1 +1,2 @@ -# in format of bundles..lastupdated= \ No newline at end of file +# in format of bundles..lastupdated= +bundles.ja.lastupdated=d9a37c48f4bd0dff014eead73a0eb730c875ed9f \ No newline at end of file From 504240a57f82ae69df8b216b7f83a83512e5effe Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 29 Jul 2020 06:55:03 -0400 Subject: [PATCH 40/42] updating documentation concerning csv path notation --- release_scripts/localization_scripts/README.md | 6 +++--- release_scripts/localization_scripts/allbundlesscript.py | 4 +++- release_scripts/localization_scripts/diffscript.py | 4 +++- release_scripts/localization_scripts/updatepropsscript.py | 5 ++++- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/release_scripts/localization_scripts/README.md b/release_scripts/localization_scripts/README.md index 408e868cf6..8d6ef1e5c2 100644 --- a/release_scripts/localization_scripts/README.md +++ b/release_scripts/localization_scripts/README.md @@ -9,12 +9,12 @@ All of these scripts provide more details on usage by calling the script with `- ## Basic Localization Update Workflow -1. Call `python3 diffscript.py -l ` to generate a csv file containing differences in properties file values from the language's previous commit to the `HEAD` commit. The language identifier should be the abbreviated identifier used for the bundle (i.e. 'ja' for Japanese). +1. Call `python3 diffscript.py -l ` to generate a csv file containing differences in properties file values from the language's previous commit to the `HEAD` commit. The language identifier should be the abbreviated identifier used for the bundle (i.e. 'ja' for Japanese). The output path should be specified as a relative path with the dot slash notation (i.e. `./outputpath.csv`) or an absolute path. 2. Update csv file with translations -3. Call `python3 updatepropsscript.py -l ` to update properties files based on the newly generated csv file. The csv file should be formatted such that the columns are bundle relative path, property files key, translated value and commit id for the latest commit id for which these changes represent. The commit id only needs to be in the header row. +3. Call `python3 updatepropsscript.py -l ` to update properties files based on the newly generated csv file. The csv file should be formatted such that the columns are bundle relative path, property files key, translated value and commit id for the latest commit id for which these changes represent. The commit id only needs to be in the header row. The output path should be specified as a relative path with the dot slash notation (i.e. `./outputpath.csv`) or an absolute path. ## Localization Generation for the First Time -First-time updates should follow a similar procedure except that instead of calling `diffscript.py`, call `python3 allbundlesscript ` to generate a csv file with relative paths of bundle files, property file keys, property file values. +First-time updates should follow a similar procedure except that instead of calling `diffscript.py`, call `python3 allbundlesscript ` to generate a csv file with relative paths of bundle files, property file keys, property file values. The output path should be specified as a relative path with the dot slash notation (i.e. `./inputpath.csv`) or an absolute path. ##Unit Tests Unit tests can be run from this directory using `python3 -m unittest`. \ No newline at end of file diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py index 38426d824a..139dec15a5 100644 --- a/release_scripts/localization_scripts/allbundlesscript.py +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -51,7 +51,9 @@ def main(): parser = argparse.ArgumentParser(description='Gathers all key-value pairs within .properties-MERGED files into ' 'one csv file.', formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument(dest='output_path', type=str, help='The path to the output csv file.') + parser.add_argument(dest='output_path', type=str, help='The path to the output csv file. The output path should be' + ' specified as a relative path with the dot slash notation ' + '(i.e. \'./outputpath.csv\') or an absolute path.') parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, help='The path to the repo. If not specified, path of script is used.') parser.add_argument('-nc', '--no_commit', dest='no_commit', action='store_true', default=False, diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index 259191292e..2713fef518 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -54,7 +54,9 @@ def main(): "'.properties-MERGED' files and generates a csv file containing " "the items changed.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument(dest='output_path', type=str, help='The path to the output csv file.') + parser.add_argument(dest='output_path', type=str, help='The path to the output csv file. The output path should ' + 'be specified as a relative path with the dot slash notation' + ' (i.e. \'./outputpath.csv\') or an absolute path.') parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, help='The path to the repo. If not specified, path of script is used.') diff --git a/release_scripts/localization_scripts/updatepropsscript.py b/release_scripts/localization_scripts/updatepropsscript.py index 2663ee330a..3d8489af82 100644 --- a/release_scripts/localization_scripts/updatepropsscript.py +++ b/release_scripts/localization_scripts/updatepropsscript.py @@ -178,7 +178,10 @@ def main(): 'deleted, and commit id for how recent these updates are. ' 'If the key should be deleted, the deletion row should be ' '\'DELETION.\' A header row is expected by default and the ' - 'commit id, if specified, should only be in the first row.') + 'commit id, if specified, should only be in the first row. The' + ' input path should be specified as a relative path with the ' + 'dot slash notation (i.e. `./inputpath.csv`) or an absolute ' + 'path.') parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, help='The path to the repo. If not specified, parent repo of path of script is used.') From 34ee2bd5c146d7423868e27f63ac6f8deabe1dbd Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 30 Jul 2020 11:15:04 -0400 Subject: [PATCH 41/42] non editable table model --- .../DataSourceSummaryCountsPanel.java | 7 ++-- .../NonEditableTableModel.java | 36 +++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/NonEditableTableModel.java diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.java index 44a108d0aa..2a1190d30b 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.java @@ -22,7 +22,6 @@ import java.util.Map; import org.sleuthkit.autopsy.coreutils.Logger; import javax.swing.JLabel; import javax.swing.table.DefaultTableCellRenderer; -import javax.swing.table.DefaultTableModel; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.FileTypeUtils; @@ -115,15 +114,15 @@ class DataSourceSummaryCountsPanel extends javax.swing.JPanel { * @param artifactDataModel The artifact type data model. */ private void updateCountsTableData(Object[][] mimeTypeDataModel, Object[][] fileCategoryDataModel, Object[][] artifactDataModel) { - fileCountsByMimeTypeTable.setModel(new DefaultTableModel(mimeTypeDataModel, MIME_TYPE_COLUMN_HEADERS)); + fileCountsByMimeTypeTable.setModel(new NonEditableTableModel(mimeTypeDataModel, MIME_TYPE_COLUMN_HEADERS)); fileCountsByMimeTypeTable.getColumnModel().getColumn(1).setCellRenderer(rightAlignedRenderer); fileCountsByMimeTypeTable.getColumnModel().getColumn(0).setPreferredWidth(130); - fileCountsByCategoryTable.setModel(new DefaultTableModel(fileCategoryDataModel, FILE_BY_CATEGORY_COLUMN_HEADERS)); + fileCountsByCategoryTable.setModel(new NonEditableTableModel(fileCategoryDataModel, FILE_BY_CATEGORY_COLUMN_HEADERS)); fileCountsByCategoryTable.getColumnModel().getColumn(1).setCellRenderer(rightAlignedRenderer); fileCountsByCategoryTable.getColumnModel().getColumn(0).setPreferredWidth(130); - artifactCountsTable.setModel(new DefaultTableModel(artifactDataModel, ARTIFACT_COUNTS_COLUMN_HEADERS)); + artifactCountsTable.setModel(new NonEditableTableModel(artifactDataModel, ARTIFACT_COUNTS_COLUMN_HEADERS)); artifactCountsTable.getColumnModel().getColumn(0).setPreferredWidth(230); artifactCountsTable.getColumnModel().getColumn(1).setCellRenderer(rightAlignedRenderer); diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/NonEditableTableModel.java b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/NonEditableTableModel.java new file mode 100644 index 0000000000..0550e8f778 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/NonEditableTableModel.java @@ -0,0 +1,36 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.casemodule.datasourcesummary; + +import javax.swing.table.DefaultTableModel; + +/** + * A Table model where cells are not editable. + */ +class NonEditableTableModel extends DefaultTableModel { + + NonEditableTableModel(Object[][] data, Object[] columnNames) { + super(data, columnNames); + } + + @Override + public boolean isCellEditable(int row, int column) { + return false; + } +} From 1bb530923df9cc4f9003556bc96216b6c7309795 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 30 Jul 2020 11:51:07 -0400 Subject: [PATCH 42/42] fix for artifact counts table --- .../datasourcesummary/DataSourceSummaryCountsPanel.form | 3 --- .../datasourcesummary/DataSourceSummaryCountsPanel.java | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.form b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.form index b82bd6f3a8..9a82746da6 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.form @@ -113,9 +113,6 @@ - - - diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.java index 2a1190d30b..6bb6603c11 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.java @@ -75,6 +75,7 @@ class DataSourceSummaryCountsPanel extends javax.swing.JPanel { initComponents(); fileCountsByMimeTypeTable.getTableHeader().setReorderingAllowed(false); fileCountsByCategoryTable.getTableHeader().setReorderingAllowed(false); + artifactCountsTable.getTableHeader().setReorderingAllowed(false); setDataSource(null); } @@ -279,7 +280,6 @@ class DataSourceSummaryCountsPanel extends javax.swing.JPanel { org.openide.awt.Mnemonics.setLocalizedText(jLabel1, org.openide.util.NbBundle.getMessage(DataSourceSummaryCountsPanel.class, "DataSourceSummaryCountsPanel.jLabel1.text")); // NOI18N - artifactCountsTable.setAutoCreateRowSorter(true); artifactCountsScrollPane.setViewportView(artifactCountsTable); javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);