From 606b41f47c6fa117b859957c744f1c35565292b2 Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Mon, 6 Jul 2020 14:59:17 -0400 Subject: [PATCH 001/102] 6516: CentralRepo accounts should be created with a normalized account identifier. --- .../datamodel/CentralRepoAccount.java | 31 +++++++++++++++++++ .../datamodel/CentralRepository.java | 6 ++-- .../datamodel/CorrelationAttributeUtil.java | 7 ++++- .../datamodel/RdbmsCentralRepo.java | 8 +++-- .../persona/Bundle.properties-MERGED | 2 ++ .../persona/CreatePersonaAccountDialog.java | 13 +++++++- 6 files changed, 60 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoAccount.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoAccount.java index 43e3c38385..6fcc8b7404 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoAccount.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoAccount.java @@ -300,4 +300,35 @@ public final class CentralRepoAccount { } return normalizedAccountIdentifier; } + + /** + * Normalizes an account identifier, based on the given account type. + * + * @param crAccountType Account type. + * @param accountIdentifier Account identifier to be normalized. + * @return Normalized identifier. + * + * @throws InvalidAccountIDException If the account identifier is invalid. + */ + public static String normalizeAccountIdentifier(CentralRepoAccountType crAccountType, String accountIdentifier) throws InvalidAccountIDException { + String normalizedAccountIdentifier; + + if (StringUtils.isBlank(accountIdentifier)) { + throw new InvalidAccountIDException("Account identifier is null or empty."); + } + try { + if (crAccountType.getAcctType().equals(Account.Type.PHONE)) { + normalizedAccountIdentifier = CorrelationAttributeNormalizer.normalizePhone(accountIdentifier); + } else if (crAccountType.getAcctType().equals(Account.Type.EMAIL)) { + normalizedAccountIdentifier = CorrelationAttributeNormalizer.normalizeEmail(accountIdentifier); + } else { + // convert to lowercase + normalizedAccountIdentifier = accountIdentifier.toLowerCase(); + } + } catch (CorrelationAttributeNormalizationException ex) { + throw new InvalidAccountIDException("Invalid account identifier", ex); + } + + return normalizedAccountIdentifier; + } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepository.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepository.java index a6af74ad3c..596ae4e90c 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepository.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepository.java @@ -27,6 +27,7 @@ import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoAccount.CentralRepoAccountType; import org.sleuthkit.autopsy.coordinationservice.CoordinationService; import org.sleuthkit.datamodel.HashHitInfo; +import org.sleuthkit.datamodel.InvalidAccountIDException; /** * Main interface for interacting with the database @@ -897,8 +898,9 @@ public interface CentralRepository { * @param accountUniqueID type specific unique account id * @return CR account * - * @throws CentralRepoException + * @throws CentralRepoException If there is an error accessing Central Repository. + * @throws InvalidAccountIDException If the account identifier is not valid. */ - CentralRepoAccount getOrCreateAccount(CentralRepoAccount.CentralRepoAccountType crAccountType, String accountUniqueID) throws CentralRepoException; + CentralRepoAccount getOrCreateAccount(CentralRepoAccount.CentralRepoAccountType crAccountType, String accountUniqueID) throws InvalidAccountIDException, CentralRepoException; } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java index cb14bf5fa2..71efaa52b8 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java @@ -35,6 +35,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.HashUtility; +import org.sleuthkit.datamodel.InvalidAccountIDException; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -187,6 +188,10 @@ public class CorrelationAttributeUtil { logger.log(Level.SEVERE, String.format("Error normalizing correlation attribute (%s)", artifact), ex); // NON-NLS return correlationAttrs; } + catch (InvalidAccountIDException ex) { + logger.log(Level.SEVERE, String.format("Invalid account identifier (%s)", artifact), ex); // NON-NLS + return correlationAttrs; + } catch (CentralRepoException ex) { logger.log(Level.SEVERE, String.format("Error querying central repository (%s)", artifact), ex); // NON-NLS return correlationAttrs; @@ -281,7 +286,7 @@ public class CorrelationAttributeUtil { * * @return The correlation attribute instance. */ - private static void makeCorrAttrFromAcctArtifact(List corrAttrInstances, BlackboardArtifact acctArtifact) throws TskCoreException, CentralRepoException { + private static void makeCorrAttrFromAcctArtifact(List corrAttrInstances, BlackboardArtifact acctArtifact) throws InvalidAccountIDException, TskCoreException, CentralRepoException { // Get the account type from the artifact BlackboardAttribute accountTypeAttribute = acctArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE)); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java index b550804421..02c9028d27 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java @@ -53,6 +53,7 @@ import org.sleuthkit.autopsy.healthmonitor.TimingMetric; import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.CaseDbSchemaVersionNumber; import org.sleuthkit.datamodel.HashHitInfo; +import org.sleuthkit.datamodel.InvalidAccountIDException; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskData; @@ -1081,16 +1082,17 @@ abstract class RdbmsCentralRepo implements CentralRepository { * within TSK core */ @Override - public CentralRepoAccount getOrCreateAccount(CentralRepoAccountType crAccountType, String accountUniqueID) throws CentralRepoException { + public CentralRepoAccount getOrCreateAccount(CentralRepoAccountType crAccountType, String accountUniqueID) throws InvalidAccountIDException, CentralRepoException { // Get the account fom the accounts table - CentralRepoAccount account = getAccount(crAccountType, accountUniqueID); + String normalizedAccountID = CentralRepoAccount.normalizeAccountIdentifier(crAccountType, accountUniqueID); + CentralRepoAccount account = getAccount(crAccountType, normalizedAccountID); // account not found in the table, create it if (null == account) { String query = "INSERT INTO accounts (account_type_id, account_unique_identifier) " + "VALUES ( " + crAccountType.getAccountTypeId() + ", '" - + accountUniqueID + "' )"; + + normalizedAccountID + "' )"; try (Connection connection = connect(); Statement s = connection.createStatement();) { diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/persona/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/centralrepository/persona/Bundle.properties-MERGED index 9a1155bbe0..c035a14196 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/persona/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/persona/Bundle.properties-MERGED @@ -6,6 +6,8 @@ AddMetadataDialog_empty_name_Title=Missing field(s) CreatePersonaAccountDialog.title.text=Create Account CreatePersonaAccountDialog_dup_msg=An account with this identifier and type already exists. CreatePersonaAccountDialog_dup_Title=Account creation failure +CreatePersonaAccountDialog_invalid_account_msg=Account identifier is not valid. +CreatePersonaAccountDialog_invalid_account_Title=Invalid account identifier CTL_OpenPersonas=Personas CTL_PersonasTopComponentAction=Personas CTL_PersonaDetailsTopComponent=Persona Details diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/persona/CreatePersonaAccountDialog.java b/Core/src/org/sleuthkit/autopsy/centralrepository/persona/CreatePersonaAccountDialog.java index 1ac9abf3b7..c477faf24a 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/persona/CreatePersonaAccountDialog.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/persona/CreatePersonaAccountDialog.java @@ -28,6 +28,7 @@ import javax.swing.JList; import javax.swing.JOptionPane; import javax.swing.ListCellRenderer; import javax.swing.SwingUtilities; +import org.openide.util.Exceptions; import org.openide.util.NbBundle.Messages; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoAccount; @@ -35,6 +36,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoAccount.Cent import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoException; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepository; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.InvalidAccountIDException; /** * Configuration dialog for creating an account. @@ -212,6 +214,9 @@ public class CreatePersonaAccountDialog extends JDialog { setVisible(true); } + @Messages({ + "CreatePersonaAccountDialog_invalid_account_Title=Invalid account identifier", + "CreatePersonaAccountDialog_invalid_account_msg=Account identifier is not valid.",}) private CentralRepoAccount createAccount(CentralRepoAccount.CentralRepoAccountType type, String identifier) { CentralRepoAccount ret = null; try { @@ -222,8 +227,14 @@ public class CreatePersonaAccountDialog extends JDialog { } catch (CentralRepoException e) { logger.log(Level.SEVERE, "Failed to access central repository", e); JOptionPane.showMessageDialog(this, - Bundle.PersonaAccountDialog_get_types_exception_Title(), Bundle.PersonaAccountDialog_get_types_exception_msg(), + Bundle.PersonaAccountDialog_get_types_exception_Title(), + JOptionPane.ERROR_MESSAGE); + } catch (InvalidAccountIDException e) { + logger.log(Level.WARNING, "Invalid account identifier", e); + JOptionPane.showMessageDialog(this, + Bundle.CreatePersonaAccountDialog_invalid_account_msg(), + Bundle.CreatePersonaAccountDialog_invalid_account_Title(), JOptionPane.ERROR_MESSAGE); } return ret; From 90fc65d2bc79da0d2ae60f6374c85855331386f7 Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Mon, 6 Jul 2020 15:22:33 -0400 Subject: [PATCH 002/102] Addressed Codacy comments. --- .../centralrepository/datamodel/CentralRepoAccount.java | 5 +++-- .../persona/CreatePersonaAccountDialog.java | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoAccount.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoAccount.java index 6fcc8b7404..eb245db07e 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoAccount.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoAccount.java @@ -311,11 +311,12 @@ public final class CentralRepoAccount { * @throws InvalidAccountIDException If the account identifier is invalid. */ public static String normalizeAccountIdentifier(CentralRepoAccountType crAccountType, String accountIdentifier) throws InvalidAccountIDException { - String normalizedAccountIdentifier; - + if (StringUtils.isBlank(accountIdentifier)) { throw new InvalidAccountIDException("Account identifier is null or empty."); } + + String normalizedAccountIdentifier; try { if (crAccountType.getAcctType().equals(Account.Type.PHONE)) { normalizedAccountIdentifier = CorrelationAttributeNormalizer.normalizePhone(accountIdentifier); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/persona/CreatePersonaAccountDialog.java b/Core/src/org/sleuthkit/autopsy/centralrepository/persona/CreatePersonaAccountDialog.java index c477faf24a..50a792b170 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/persona/CreatePersonaAccountDialog.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/persona/CreatePersonaAccountDialog.java @@ -28,7 +28,6 @@ import javax.swing.JList; import javax.swing.JOptionPane; import javax.swing.ListCellRenderer; import javax.swing.SwingUtilities; -import org.openide.util.Exceptions; import org.openide.util.NbBundle.Messages; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoAccount; From 202721acae8155da3214e03438726979c9531e21 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 10 Jul 2020 13:02:45 -0400 Subject: [PATCH 003/102] adding localization helper scripts --- release_scripts/diffscript.py | 220 ---------------- .../localization_scripts/.gitignore | 2 + .../localization_scripts/allbundlesscript.py | 43 ++++ .../localization_scripts/csvutil.py | 47 ++++ .../localization_scripts/diffscript.py | 61 +++++ .../localization_scripts/gitutil.py | 147 +++++++++++ .../localization_scripts/itemchange.py | 96 +++++++ .../localization_scripts/propentry.py | 14 + .../localization_scripts/propsutil.py | 65 +++++ .../localization_scripts/updatepropsscript.py | 240 ++++++++++++++++++ 10 files changed, 715 insertions(+), 220 deletions(-) delete mode 100644 release_scripts/diffscript.py create mode 100644 release_scripts/localization_scripts/.gitignore create mode 100644 release_scripts/localization_scripts/allbundlesscript.py create mode 100644 release_scripts/localization_scripts/csvutil.py create mode 100644 release_scripts/localization_scripts/diffscript.py create mode 100644 release_scripts/localization_scripts/gitutil.py create mode 100644 release_scripts/localization_scripts/itemchange.py create mode 100644 release_scripts/localization_scripts/propentry.py create mode 100644 release_scripts/localization_scripts/propsutil.py create mode 100644 release_scripts/localization_scripts/updatepropsscript.py diff --git a/release_scripts/diffscript.py b/release_scripts/diffscript.py deleted file mode 100644 index af3d995a88..0000000000 --- a/release_scripts/diffscript.py +++ /dev/null @@ -1,220 +0,0 @@ -"""This script determines the updated, added, and deleted properties from the '.properties-MERGED' files -and generates a csv file containing the items changed. This script requires the python libraries: -gitpython and jproperties. As a consequence, it also requires git >= 1.7.0 and python >= 3.4. -""" - -from git import Repo -from typing import List, Dict, Tuple -import re -import csv -from jproperties import Properties -import sys - - -class ItemChange: - def __init__(self, rel_path: str, key: str, prev_val: str, cur_val: str): - """Describes the change that occurred for a particular key of a properties file. - - Args: - rel_path (str): The relative path of the properties file. - key (str): The key in the properties file. - prev_val (str): The previous value for the key. - cur_val (str): The current value for the key. - """ - self.rel_path = rel_path - self.key = key - self.prev_val = prev_val - self.cur_val = cur_val - if ItemChange.has_str_content(cur_val) and not ItemChange.has_str_content(prev_val): - self.type = 'ADDITION' - elif not ItemChange.has_str_content(cur_val) and ItemChange.has_str_content(prev_val): - self.type = 'DELETION' - else: - self.type = 'CHANGE' - - @staticmethod - def has_str_content(content: str): - """Determines whether or not the content is empty or None. - - Args: - content (str): The text. - - Returns: - bool: Whether or not it has content. - """ - return content is not None and len(content.strip()) > 0 - - @staticmethod - def get_headers() -> List[str]: - """Returns the csv headers to insert when serializing a list of ItemChange objects to csv. - - Returns: - List[str]: The column headers - """ - return ['Relative Path', 'Key', 'Change Type', 'Previous Value', 'Current Value'] - - def get_row(self) -> List[str]: - """Returns the list of values to be entered as a row in csv serialization. - - Returns: - List[str]: The list of values to be entered as a row in csv serialization. - """ - return [ - self.rel_path, - self.key, - self.type, - self.prev_val, - self.cur_val] - - -def get_entry_dict(diff_str: str) -> Dict[str, str]: - """Retrieves a dictionary mapping the properties represented in the string. - - Args: - diff_str (str): The string of the properties file. - - Returns: - Dict[str,str]: The mapping of keys to values in that properties file. - """ - props = Properties() - props.load(diff_str, "utf-8") - return props.properties - - -def get_item_change(rel_path: str, key: str, prev_val: str, cur_val: str) -> ItemChange: - """Returns an ItemChange object if the previous value is not equal to the current value. - - Args: - rel_path (str): The relative path for the properties file. - key (str): The key within the properties file for this potential change. - prev_val (str): The previous value. - cur_val (str): The current value. - - Returns: - ItemChange: The ItemChange object or None if values are the same. - """ - if (prev_val == cur_val): - return None - else: - return ItemChange(rel_path, key, prev_val, cur_val) - - -def get_changed(rel_path: str, a_str: str, b_str: str) -> List[ItemChange]: - """Given the relative path of the properties file that - - Args: - rel_path (str): The relative path for the properties file. - a_str (str): The string representing the original state of the file. - b_str (str): The string representing the current state of the file. - - Returns: - List[ItemChange]: The changes determined. - """ - print('Retrieving changes for {}...'.format(rel_path)) - a_dict = get_entry_dict(a_str) - b_dict = get_entry_dict(b_str) - all_keys = set().union(a_dict.keys(), b_dict.keys()) - mapped = map(lambda key: get_item_change( - rel_path, key, a_dict.get(key), b_dict.get(key)), all_keys) - return filter(lambda entry: entry is not None, mapped) - - -def get_text(blob) -> str: - return blob.data_stream.read().decode('utf-8') - - -def get_changed_from_diff(rel_path: str, diff) -> List[ItemChange]: - """Determines changes from a git python diff. - - Args: - rel_path (str): The relative path for the properties file. - diff: The git python diff. - - Returns: - List[ItemChange]: The changes in properties. - """ - # an item was added - if diff.change_type == 'A': - changes = get_changed(rel_path, '', get_text(diff.b_blob)) - # an item was deleted - elif diff.change_type == 'D': - changes = get_changed(rel_path, get_text(diff.a_blob), '') - # an item was modified - elif diff.change_type == 'M': - changes = get_changed(rel_path, get_text( - diff.a_blob), get_text(diff.b_blob)) - else: - changes = [] - - return changes - - -def get_rel_path(diff) -> str: - """Determines the relative path based on the git python. - - Args: - diff: The git python diff. - - Returns: - str: The determined relative path. - """ - if diff.b_path is not None: - return diff.b_path - elif diff.a_path is not None: - return diff.a_path - else: - return '' - - -def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit_2_id: str): - """Determines the changes made in '.properties-MERGED' files from one commit to another commit. - - Args: - repo_path (str): The local path to the git repo. - output_path (str): The output path for the csv file. - commit_1_id (str): The initial commit for the diff. - commit_2_id (str): The latest commit for the diff. - """ - repo = Repo(repo_path) - commit_1 = repo.commit(commit_1_id) - commit_2 = repo.commit(commit_2_id) - - diffs = commit_1.diff(commit_2) - with open(output_path, 'w', newline='') as csvfile: - writer = csv.writer(csvfile) - writer.writerow(ItemChange.get_headers()) - - for diff in diffs: - rel_path = get_rel_path(diff) - if not rel_path.endswith('.properties-MERGED'): - continue - - changes = get_changed_from_diff(rel_path, diff) - - for item_change in changes: - writer.writerow(item_change.get_row()) - - -def print_help(): - """Prints a quick help message. - """ - print("diffscript.py [path to repo] [csv output path] [commit for previous release] [commit for current release (optional; defaults to 'HEAD')]") - - -def main(): - if len(sys.argv) <= 3: - print_help() - sys.exit(1) - - repo_path = sys.argv[1] - output_path = sys.argv[2] - commit_1_id = sys.argv[3] - commit_2_id = sys.argv[4] if len(sys.argv) > 4 else 'HEAD' - - write_diff_to_csv(repo_path, output_path, commit_1_id, commit_2_id) - - sys.exit(0) - - -if __name__ == "__main__": - main() diff --git a/release_scripts/localization_scripts/.gitignore b/release_scripts/localization_scripts/.gitignore new file mode 100644 index 0000000000..341e31ecbc --- /dev/null +++ b/release_scripts/localization_scripts/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +.idea \ No newline at end of file diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py new file mode 100644 index 0000000000..80fa767430 --- /dev/null +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -0,0 +1,43 @@ +"""This script finds all '.properties-MERGED' files and writes relative path, key, and value to a CSV file. +This script requires the python libraries: gitpython and jproperties. As a consequence, it also requires +git >= 1.7.0 and python >= 3.4. +""" + +import sys +from gitutil import get_property_file_entries +from csvutil import records_to_csv +import argparse + + +def write_items_to_csv(repo_path: str, output_path: str): + """Determines the contents of '.properties-MERGED' files and writes to a csv file. + + Args: + repo_path (str): The local path to the git repo. + output_path (str): The output path for the csv file. + """ + + rows = [['Relative path', 'Key', 'Value']] + for entry in get_property_file_entries(repo_path): + rows.append([entry.rel_path, entry.key, entry.value]) + + records_to_csv(output_path, rows) + + +def main(): + parser = argparse.ArgumentParser(description='Gathers all key-value pairs within .properties-MERGED files into ' + + 'one csv file.') + parser.add_argument(dest='repo_path', type=str, help='The path to the repo.') + parser.add_argument(dest='output_path', type=str, help='The path to the output csv file.') + + args = parser.parse_args() + repo_path = args.repo_path + output_path = args.output_path + + write_items_to_csv(repo_path, output_path) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/release_scripts/localization_scripts/csvutil.py b/release_scripts/localization_scripts/csvutil.py new file mode 100644 index 0000000000..5e0309998e --- /dev/null +++ b/release_scripts/localization_scripts/csvutil.py @@ -0,0 +1,47 @@ +"""Provides tools for parsing and writing to a csv file. +""" + +from typing import List, Iterable, Iterator +import csv +import os + + +def records_to_csv(output_path: str, rows: Iterable[List[str]]): + """Writes rows to a csv file at the specified path. + + Args: + output_path (str): The path where the csv file will be written. + rows (List[List[str]]): The rows to be written. Each row of a + list of strings will be written according + to their index (i.e. column 3 will be index 2). + """ + + parent_dir, file = os.path.split(output_path) + if not os.path.exists(parent_dir): + os.makedirs(parent_dir) + + with open(output_path, 'w', encoding="utf-8", newline='') as csvfile: + writer = csv.writer(csvfile) + + for row in rows: + writer.writerow(row) + + +def csv_to_records(input_path: str, header_row: bool) -> Iterator[List[str]]: + """Writes rows to a csv file at the specified path. + + Args: + input_path (str): The path where the csv file will be written. + header_row (bool): The rows to be written. Each row of a + list of strings will be written according + to their index (i.e. column 3 will be index 2). + """ + + with open(input_path, encoding='utf-8') as csv_file: + csv_reader = csv.reader(csv_file, delimiter=',') + + for row in csv_reader: + if header_row: + header_row = False + else: + yield row diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py new file mode 100644 index 0000000000..d7b679d9b3 --- /dev/null +++ b/release_scripts/localization_scripts/diffscript.py @@ -0,0 +1,61 @@ +"""This script determines the updated, added, and deleted properties from the '.properties-MERGED' files +and generates a csv file containing the items changed. This script requires the python libraries: +gitpython and jproperties. As a consequence, it also requires git >= 1.7.0 and python >= 3.4. +""" + +import sys +from gitutil import get_property_files_diff, get_commit_id +from itemchange import ItemChange +from csvutil import records_to_csv +import argparse + + +def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit_2_id: str, show_commits: bool): + """Determines the changes made in '.properties-MERGED' files from one commit to another commit. + + Args: + repo_path (str): The local path to the git repo. + output_path (str): The output path for the csv file. + commit_1_id (str): The initial commit for the diff. + commit_2_id (str): The latest commit for the diff. + show_commits (bool): show commits in the header row. + """ + + row_header = ItemChange.get_headers() + if show_commits: + row_header += [get_commit_id(repo_path, commit_1_id), get_commit_id(repo_path, commit_2_id)] + + rows = [row_header] + + rows += map(lambda item_change: item_change.get_row(), + get_property_files_diff(repo_path, commit_1_id, commit_2_id)) + + records_to_csv(output_path, rows) + + +def main(): + parser = argparse.ArgumentParser(description="determines the updated, added, and deleted properties from the " + + "'.properties-MERGED' files and generates a csv file containing " + + "the items changed.") + parser.add_argument(dest='repo_path', type=str, help='The path to the repo.') + parser.add_argument(dest='output_path', type=str, help='The path to the output csv file.') + parser.add_argument(dest='commit_1_id', type=str, help='The commit for previous release.') + parser.add_argument('-lc', '--latest_commit', dest='commit_2_id', type=str, default='HEAD', required=False, + help='The commit for current release.') + parser.add_argument('-nc', '--no_commits', dest='no_commits', action='store_true', default=False, + required=False, help="Suppresses adding commits to the generated csv header.") + + args = parser.parse_args() + repo_path = args.repo_path + output_path = args.output_path + commit_1_id = args.commit_1_id + commit_2_id = args.commit_2_id + show_commits = not args.no_commits + + write_diff_to_csv(repo_path, output_path, commit_1_id, commit_2_id, show_commits) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/release_scripts/localization_scripts/gitutil.py b/release_scripts/localization_scripts/gitutil.py new file mode 100644 index 0000000000..40b9a15adb --- /dev/null +++ b/release_scripts/localization_scripts/gitutil.py @@ -0,0 +1,147 @@ +from git import Repo, Diff, Blob +from typing import List, Union, Iterator, Tuple, Any +from itemchange import ItemChange, get_changed +from pathlib import Path +from propentry import PropEntry +from propsutil import DEFAULT_PROPS_EXTENSION, get_entry_dict + + +def get_text(blob: Blob) -> str: + return blob.data_stream.read().decode('utf-8') + + +def get_changed_from_diff(rel_path: str, diff: Diff) -> List[ItemChange]: + """Determines changes from a git python diff. + + Args: + rel_path (str): The relative path for the properties file. + diff (Diff): The git python diff. + + Returns: + List[ItemChange]: The changes in properties. + """ + + # an item was added + if diff.change_type == 'A': + changes = get_changed(rel_path, '', get_text(diff.b_blob)) + # an item was deleted + elif diff.change_type == 'D': + changes = get_changed(rel_path, get_text(diff.a_blob), '') + # an item was modified + elif diff.change_type == 'M': + changes = get_changed(rel_path, get_text( + diff.a_blob), get_text(diff.b_blob)) + else: + changes = [] + + return changes + + +def get_rel_path(diff: Diff) -> Union[str, None]: + """Determines the relative path based on the git python. + + Args: + diff: The git python diff. + + Returns: + str: The determined relative path. + """ + if diff.b_path is not None: + return diff.b_path + elif diff.a_path is not None: + return diff.a_path + else: + return None + + +def get_diff(repo_path: str, commit_1_id: str, commit_2_id: str) -> Any: + """Determines the diff between two commits. + + Args: + repo_path (str): The local path to the git repo. + commit_1_id (str): The initial commit for the diff. + commit_2_id (str): The latest commit for the diff. + + Returns: + The determined diff. + """ + repo = Repo(repo_path) + commit_1 = repo.commit(commit_1_id) + commit_2 = repo.commit(commit_2_id) + return commit_1.diff(commit_2) + + +def get_commit_id(repo_path: str, commit_id: str) -> str: + """Determines the hash for head commit. This does things like fetch the id of head if 'HEAD' is provided. + + Args: + repo_path: The path to the repo. + commit_id: The id for the commit. + + Returns: + The hash for the commit in the repo. + """ + repo = Repo(repo_path) + commit = repo.commit(commit_id) + return str(commit.hexsha) + + +def get_property_files_diff(repo_path: str, commit_1_id: str, commit_2_id: str, + property_file_extension: str = DEFAULT_PROPS_EXTENSION) -> Iterator[ItemChange]: + """Determines the item changes within property files as a diff between two commits. + + Args: + repo_path (str): The repo path. + commit_1_id (str): The first git commit. + commit_2_id (str): The second git commit. + property_file_extension (str): The extension for properties files to gather. + + Returns: + All found item changes in values of keys between the property files. + """ + + diffs = get_diff(repo_path, commit_1_id, commit_2_id) + for diff in diffs: + rel_path = get_rel_path(diff) + if rel_path is None or not rel_path.endswith('.' + property_file_extension): + continue + + yield from get_changed_from_diff(rel_path, diff) + + +def list_paths(root_tree, path: Path = Path('.')) -> Iterator[Tuple[str, Blob]]: + """ + Given the root path to serve as a prefix, walks the tree of a git commit returning all files and blobs. + Args: + root_tree: The tree of the commit to walk. + path: The path to use as a prefix. + + Returns: A tuple iterator where each tuple consists of the path as a string and a blob of the file. + + """ + for blob in root_tree.blobs: + ret_item = (str(path / blob.name), blob) + yield ret_item + for tree in root_tree.trees: + yield from list_paths(tree, path / tree.name) + + +def get_property_file_entries(repo_path: str, at_commit: str = 'HEAD', + property_file_extension: str = DEFAULT_PROPS_EXTENSION) -> Iterator[PropEntry]: + """ + Retrieves all property files entries returning as an iterator of PropEntry objects. + Args: + repo_path: The path to the git repo. + at_commit: The commit to use. + property_file_extension: The extension to use for scanning for property files. + + Returns: An iterator of PropEntry objects. + + """ + repo = Repo(repo_path) + commit = repo.commit(at_commit) + for item in list_paths(commit.tree): + path, blob = item + if path.endswith(property_file_extension): + for key, val in get_entry_dict(get_text(blob)).items(): + yield PropEntry(path, key, val) diff --git a/release_scripts/localization_scripts/itemchange.py b/release_scripts/localization_scripts/itemchange.py new file mode 100644 index 0000000000..8533be9469 --- /dev/null +++ b/release_scripts/localization_scripts/itemchange.py @@ -0,0 +1,96 @@ +from typing import Iterator, List, Union +from propsutil import get_entry_dict + + +class ItemChange: + def __init__(self, rel_path: str, key: str, prev_val: str, cur_val: str): + """Describes the change that occurred for a particular key of a properties file. + + Args: + rel_path (str): The relative path of the properties file. + key (str): The key in the properties file. + prev_val (str): The previous value for the key. + cur_val (str): The current value for the key. + """ + self.rel_path = rel_path + self.key = key + self.prev_val = prev_val + self.cur_val = cur_val + if ItemChange.has_str_content(cur_val) and not ItemChange.has_str_content(prev_val): + self.type = 'ADDITION' + elif not ItemChange.has_str_content(cur_val) and ItemChange.has_str_content(prev_val): + self.type = 'DELETION' + else: + self.type = 'CHANGE' + + @staticmethod + def has_str_content(content: str): + """Determines whether or not the content is empty or None. + + Args: + content (str): The text. + + Returns: + bool: Whether or not it has content. + """ + return content is not None and len(content.strip()) > 0 + + @staticmethod + def get_headers() -> List[str]: + """Returns the csv headers to insert when serializing a list of ItemChange objects to csv. + + Returns: + List[str]: The column headers + """ + return ['Relative Path', 'Key', 'Change Type', 'Previous Value', 'Current Value'] + + def get_row(self) -> List[str]: + """Returns the list of values to be entered as a row in csv serialization. + + Returns: + List[str]: The list of values to be entered as a row in csv serialization. + """ + return [ + self.rel_path, + self.key, + self.type, + self.prev_val, + self.cur_val] + + +def get_item_change(rel_path: str, key: str, prev_val: str, cur_val: str) -> Union[ItemChange, None]: + """Returns an ItemChange object if the previous value is not equal to the current value. + + Args: + rel_path (str): The relative path for the properties file. + key (str): The key within the properties file for this potential change. + prev_val (str): The previous value. + cur_val (str): The current value. + + Returns: + ItemChange: The ItemChange object or None if values are the same. + """ + if prev_val == cur_val: + return None + else: + return ItemChange(rel_path, key, prev_val, cur_val) + + +def get_changed(rel_path: str, a_str: str, b_str: str) -> Iterator[ItemChange]: + """Given the relative path of the properties file that + + Args: + rel_path (str): The relative path for the properties file. + a_str (str): The string representing the original state of the file. + b_str (str): The string representing the current state of the file. + + Returns: + List[ItemChange]: The changes determined. + """ + print('Retrieving changes for {}...'.format(rel_path)) + a_dict = get_entry_dict(a_str) + b_dict = get_entry_dict(b_str) + all_keys = set().union(a_dict.keys(), b_dict.keys()) + mapped = map(lambda key: get_item_change( + rel_path, key, a_dict.get(key), b_dict.get(key)), all_keys) + return filter(lambda entry: entry is not None, mapped) diff --git a/release_scripts/localization_scripts/propentry.py b/release_scripts/localization_scripts/propentry.py new file mode 100644 index 0000000000..4821f4bc5f --- /dev/null +++ b/release_scripts/localization_scripts/propentry.py @@ -0,0 +1,14 @@ +class PropEntry: + def __init__(self, rel_path: str, key: str, value: str, should_delete: bool = False): + """Defines a property file entry to be updated in a property file. + + Args: + rel_path (str): The relative path for the property file. + key (str): The key for the entry. + value (str): The value for the entry. + should_delete (bool, optional): Whether or not the key should simply be deleted. Defaults to False. + """ + self.rel_path = rel_path + self.key = key + self.value = value + self.should_delete = should_delete diff --git a/release_scripts/localization_scripts/propsutil.py b/release_scripts/localization_scripts/propsutil.py new file mode 100644 index 0000000000..b0b301c714 --- /dev/null +++ b/release_scripts/localization_scripts/propsutil.py @@ -0,0 +1,65 @@ +"""Provides tools for reading from and writing to java properties files. +""" + +from typing import Dict, Union, IO +from jproperties import Properties +import os +from os import path + +# The default extension for property files in autopsy repo +DEFAULT_PROPS_EXTENSION = 'properties-MERGED' + + +def get_entry_dict(file_contents: Union[str, IO]) -> Dict[str, str]: + """Retrieves a dictionary mapping the properties represented in the string. + + Args: + file_contents: The string of the properties file or the file handle. + + Returns: + Dict[str,str]: The mapping of keys to values in that properties file. + """ + + props = Properties() + props.load(file_contents, "utf-8") + return props.properties + + +def set_entry_dict(contents: Dict[str, str], file_path: str): + """Sets the property file to the key-value pairs of the contents dictionary. + + Args: + contents (Dict[str, str]): The dictionary whose contents will be the key value pairs of the properties file. + file_path (str): The path to the properties file to create. + """ + + props = Properties() + for key, val in contents.items(): + props[key] = val + + parent_dir, file = os.path.split(file_path) + if not os.path.exists(parent_dir): + os.makedirs(parent_dir) + + with open(file_path, "wb") as f: + props.store(f) + + +def update_entry_dict(contents: Dict[str, str], file_path: str): + """Updates the properties file at the given location with the key-value properties of contents. + Creates a new properties file at given path if none exists. + + Args: + contents (Dict[str, str]): The dictionary whose contents will be the key value pairs of the properties file. + file_path (str): The path to the properties file to create. + """ + contents_to_edit = contents.copy() + + if path.isfile(file_path): + cur_dict = get_entry_dict(file_path) + for cur_key, cur_val in cur_dict.values(): + # only update contents if contents does not already have key + if cur_key not in contents_to_edit: + contents_to_edit[cur_key] = cur_val + + set_entry_dict(contents_to_edit, file_path) diff --git a/release_scripts/localization_scripts/updatepropsscript.py b/release_scripts/localization_scripts/updatepropsscript.py new file mode 100644 index 0000000000..f575ee847d --- /dev/null +++ b/release_scripts/localization_scripts/updatepropsscript.py @@ -0,0 +1,240 @@ +"""This script finds all '.properties-MERGED' files and writes relative path, key, and value to a CSV file. +This script requires the python libraries: jproperties. It also requires Python 3.x. +""" + +from typing import List, Dict, Tuple, Callable, Iterator +import sys +import os +from propsutil import set_entry_dict, get_entry_dict +from csvutil import csv_to_records +from propentry import PropEntry +import argparse + + +def write_prop_entries(entries: Iterator[PropEntry], repo_path: str): + """Writes property entry items to their expected relative path within the repo path. + Previously existing files will be overwritten and prop entries marked as should_be_deleted will + not be included. + + Args: + entries (List[PropEntry]): the prop entry items to write to disk. + repo_path (str): The path to the git repo. + """ + items_by_file = get_by_file(entries) + for rel_path, (entries, ignored) in items_by_file.items(): + abs_path = os.path.join(repo_path, rel_path) + set_entry_dict(entries, abs_path) + + +def update_prop_entries(entries: Iterator[PropEntry], repo_path: str): + """Updates property entry items to their expected relative path within the repo path. The union of + entries provided and any previously existing entries will be created. Keys marked for deletion will be + removed from the generated property files. + + Args: + entries (List[PropEntry]): the prop entry items to write to disk. + repo_path (str): The path to the git repo. + """ + items_by_file = get_by_file(entries) + for rel_path, (entries, to_delete) in items_by_file.items(): + abs_path = os.path.join(repo_path, rel_path) + + if os.path.isfile(abs_path): + with open(abs_path, "rb") as f: + prop_items = get_entry_dict(f) + else: + prop_items = {} + + for key_to_delete in to_delete: + if key_to_delete in prop_items: + del prop_items[key_to_delete] + + for key, val in entries.items(): + prop_items[key] = val + + set_entry_dict(prop_items, abs_path) + + +def get_by_file(entries: Iterator[PropEntry]) -> Dict[str, Tuple[Dict[str, str], List[str]]]: + """Sorts a prop entry list by file. The return type is a dictionary mapping + the file path to a tuple containing the key-value pairs to be updated and a + list of keys to be deleted. + + Args: + entries (List[PropEntry]): The entries to be sorted. + + Returns: + Dict[str, Tuple[Dict[str,str], List[str]]]: A dictionary mapping + the file path to a tuple containing the key-value pairs to be updated and a + list of keys to be deleted. + """ + to_ret = {} + for prop_entry in entries: + rel_path = prop_entry.rel_path + key = prop_entry.key + value = prop_entry.value + + if rel_path not in to_ret: + to_ret[rel_path] = ({}, []) + + if prop_entry.should_delete: + to_ret[rel_path][1].append(prop_entry.key) + else: + to_ret[rel_path][0][key] = value + + return to_ret + + +def idx_bounded(num: int, max_exclusive: int) -> bool: + return 0 <= num < max_exclusive + + +def get_prop_entry(row: List[str], + path_idx: int = 0, + key_idx: int = 1, + value_idx: int = 2, + should_delete_converter: Callable[[List[str]], bool] = None, + path_converter: Callable[[str], str] = None) -> PropEntry: + """Parses a PropEntry object from a row of values in a csv. + + Args: + row (List[str]): The csv file row to parse. + path_idx (int, optional): The column index for the relative path of the properties file. Defaults to 0. + key_idx (int, optional): The column index for the properties key. Defaults to 1. + value_idx (int, optional): The column index for the properties value. Defaults to 2. + should_delete_converter (Callable[[List[str]], bool], optional): If not None, this determines if the key should + be deleted from the row values. Defaults to None. + path_converter (Callable[[str], str], optional): If not None, this determines the relative path to use in the + created PropEntry given the original relative path. Defaults to None. + + Returns: + PropEntry: The generated prop entry object. + """ + + path = row[path_idx] if idx_bounded(path_idx, len(row)) else None + if path_converter is not None: + path = path_converter(path) + + key = row[key_idx] if idx_bounded(key_idx, len(row)) else None + value = row[value_idx] if idx_bounded(value_idx, len(row)) else None + should_delete = False if should_delete_converter is None else should_delete_converter(row) + return PropEntry(path, key, value, should_delete) + + +def get_prop_entries(rows: List[List[str]], + path_idx: int = 0, + key_idx: int = 1, + value_idx: int = 2, + should_delete_converter: Callable[[List[str]], bool] = None, + path_converter: Callable[[str], str] = None) -> Iterator[PropEntry]: + + """Parses PropEntry objects from rows of values in a csv. + + Args: + rows (List[List[str]]): The csv file rows to parse. + path_idx (int, optional): The column index for the relative path of the properties file. Defaults to 0. + key_idx (int, optional): The column index for the properties key. Defaults to 1. + value_idx (int, optional): The column index for the properties value. Defaults to 2. + should_delete_converter (Callable[[List[str]], bool], optional): If not None, this determines if the key should + be deleted from the row values. Defaults to None. + path_converter (Callable[[str], str], optional): If not None, this determines the relative path to use in the + created PropEntry given the original relative path. Defaults to None. + + Returns: + List[PropEntry]: The generated prop entry objects. + """ + return map(lambda row: get_prop_entry( + row, path_idx, key_idx, value_idx, should_delete_converter, path_converter), + rows) + + +def get_should_deleted(row_items: List[str], requested_idx: int) -> bool: + """If there is a value at row_items[requested_idx] and that value is not empty, then this will return true. + + Args: + row_items (List[str]): The row items. + requested_idx (int): The index specifying if the property should be deleted. + + Returns: + bool: True if the row specifies it should be deleted. + """ + if idx_bounded(requested_idx, len(row_items)) and len((row_items[requested_idx].strip())) > 0: + return True + else: + return False + + +def get_new_rel_path(orig_path: str, new_filename: str) -> str: + """Obtains a new relative path. This tries to determine if the provided path is a directory or filename (has an + extension containing '.') then constructs the new path with the old parent directory and the new filename. + + Args: + orig_path (str): The original path. + new_filename (str): The new filename to use. + + Returns: + str: The new path. + """ + potential_parent_dir, orig_file = os.path.split(orig_path) + parent_dir = orig_path if '.' not in orig_file else potential_parent_dir + return os.path.join(parent_dir, new_filename) + + +def main(): + parser = argparse.ArgumentParser(description='Updates properties files in the autopsy git repo.') + + parser.add_argument(dest='csv_file', type=str, help='The path to the csv file.') + parser.add_argument(dest='repo_path', type=str, help='The path to the repo.') + parser.add_argument('-p', '--path_idx', dest='path_idx', action='store', type=int, default=0, required=False, + help='The column index in the csv file providing the relative path to the properties file.') + parser.add_argument('-k', '--key_idx', dest='key_idx', action='store', type=int, default=1, required=False, + help='The column index in the csv file providing the key within the properties file.') + parser.add_argument('-v', '--value_idx', dest='value_idx', action='store', type=int, default=2, required=False, + help='The column index in the csv file providing the value within the properties file.') + parser.add_argument('-d', '--should_delete_idx', dest='should_delete_idx', action='store', type=int, default=None, + required=False, help='The column index in the csv file providing whether or not the file ' + + 'should be deleted. Any non-blank content will be treated as True.') + parser.add_argument('-f', '--file_rename', dest='file_rename', action='store', type=str, default=None, + required=False, help='If specified, the properties file will be renamed to the argument' + + ' preserving the specified relative path.') + parser.add_argument('--has_no_header', dest='has_no_header', action='store_true', default=False, required=False, + help='Specify whether or not there is a header within the csv file.') + parser.add_argument('-o', '--should_overwrite', dest='should_overwrite', action='store_true', default=False, + required=False, help="Whether or not to overwrite the previously existing properties files" + + " ignoring previously existing values.") + + args = parser.parse_args() + + repo_path = args.repo_path + input_path = args.csv_file + path_idx = args.path_idx + key_idx = args.key_idx + value_idx = args.value_idx + has_header = not args.has_no_header + overwrite = args.should_overwrite + + if args.should_delete_idx is None: + should_delete_converter = None + else: + def should_delete_converter(row_items: List[str]): + return get_should_deleted(row_items, args.should_delete_idx) + + if args.file_rename is None: + path_converter = None + else: + def path_converter(orig_path: str): + return get_new_rel_path(orig_path, args.file_rename) + + all_items = list(csv_to_records(input_path, has_header)) + prop_entries = get_prop_entries(all_items, path_idx, key_idx, value_idx, should_delete_converter, path_converter) + + if overwrite: + write_prop_entries(prop_entries, repo_path) + else: + update_prop_entries(prop_entries, repo_path) + + sys.exit(0) + + +if __name__ == "__main__": + main() From bb8d370a3f803e8261d5caa4422ffed4482f8830 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 10 Jul 2020 14:31:18 -0400 Subject: [PATCH 004/102] commits in headers --- .../localization_scripts/allbundlesscript.py | 19 ++++++++++++++----- .../localization_scripts/gitutil.py | 3 +++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py index 80fa767430..c4273a7a5d 100644 --- a/release_scripts/localization_scripts/allbundlesscript.py +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -1,23 +1,29 @@ """This script finds all '.properties-MERGED' files and writes relative path, key, and value to a CSV file. This script requires the python libraries: gitpython and jproperties. As a consequence, it also requires -git >= 1.7.0 and python >= 3.4. +git >= 1.7.0 and python >= 3.4. This script relies on fetching 'HEAD' from current branch. So make sure +repo is on correct branch (i.e. develop). """ import sys -from gitutil import get_property_file_entries +from gitutil import get_property_file_entries, get_commit_id from csvutil import records_to_csv import argparse -def write_items_to_csv(repo_path: str, output_path: str): +def write_items_to_csv(repo_path: str, output_path: str, show_commit: bool): """Determines the contents of '.properties-MERGED' files and writes to a csv file. Args: repo_path (str): The local path to the git repo. output_path (str): The output path for the csv file. + show_commit (bool): Whether or not to include the commit id in the header """ - rows = [['Relative path', 'Key', 'Value']] + row_header = ['Relative path', 'Key', 'Value'] + if show_commit: + row_header.append(get_commit_id(repo_path, 'HEAD')) + + rows = [row_header] for entry in get_property_file_entries(repo_path): rows.append([entry.rel_path, entry.key, entry.value]) @@ -29,12 +35,15 @@ def main(): 'one csv file.') parser.add_argument(dest='repo_path', type=str, help='The path to the repo.') parser.add_argument(dest='output_path', type=str, help='The path to the output csv file.') + parser.add_argument('-nc', '--no_commit', dest='no_commit', action='store_true', default=False, + required=False, help="Suppresses adding commits to the generated csv header.") args = parser.parse_args() repo_path = args.repo_path output_path = args.output_path + show_commit = not args.no_commit - write_items_to_csv(repo_path, output_path) + write_items_to_csv(repo_path, output_path, show_commit) sys.exit(0) diff --git a/release_scripts/localization_scripts/gitutil.py b/release_scripts/localization_scripts/gitutil.py index 40b9a15adb..dd7b13632f 100644 --- a/release_scripts/localization_scripts/gitutil.py +++ b/release_scripts/localization_scripts/gitutil.py @@ -112,6 +112,8 @@ def get_property_files_diff(repo_path: str, commit_1_id: str, commit_2_id: str, def list_paths(root_tree, path: Path = Path('.')) -> Iterator[Tuple[str, Blob]]: """ Given the root path to serve as a prefix, walks the tree of a git commit returning all files and blobs. + Repurposed from: https://www.enricozini.org/blog/2019/debian/gitpython-list-all-files-in-a-git-commit/ + Args: root_tree: The tree of the commit to walk. path: The path to use as a prefix. @@ -130,6 +132,7 @@ def get_property_file_entries(repo_path: str, at_commit: str = 'HEAD', property_file_extension: str = DEFAULT_PROPS_EXTENSION) -> Iterator[PropEntry]: """ Retrieves all property files entries returning as an iterator of PropEntry objects. + Args: repo_path: The path to the git repo. at_commit: The commit to use. From 54cfd1cd51dbdca706647890342615a277539254 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 13 Jul 2020 10:15:29 -0400 Subject: [PATCH 005/102] use parent git repos --- release_scripts/localization_scripts/gitutil.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release_scripts/localization_scripts/gitutil.py b/release_scripts/localization_scripts/gitutil.py index dd7b13632f..65db268fc8 100644 --- a/release_scripts/localization_scripts/gitutil.py +++ b/release_scripts/localization_scripts/gitutil.py @@ -65,7 +65,7 @@ def get_diff(repo_path: str, commit_1_id: str, commit_2_id: str) -> Any: Returns: The determined diff. """ - repo = Repo(repo_path) + repo = Repo(repo_path, search_parent_directories=True) commit_1 = repo.commit(commit_1_id) commit_2 = repo.commit(commit_2_id) return commit_1.diff(commit_2) @@ -81,7 +81,7 @@ def get_commit_id(repo_path: str, commit_id: str) -> str: Returns: The hash for the commit in the repo. """ - repo = Repo(repo_path) + repo = Repo(repo_path, search_parent_directories=True) commit = repo.commit(commit_id) return str(commit.hexsha) @@ -141,7 +141,7 @@ def get_property_file_entries(repo_path: str, at_commit: str = 'HEAD', Returns: An iterator of PropEntry objects. """ - repo = Repo(repo_path) + repo = Repo(repo_path, search_parent_directories=True) commit = repo.commit(at_commit) for item in list_paths(commit.tree): path, blob = item From f24a5a30e2d7b10516ab342317473913c55a2780 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 13 Jul 2020 11:39:18 -0400 Subject: [PATCH 006/102] working through commit args --- .../localization_scripts/diffscript.py | 29 +++++++++++++++---- .../localization_scripts/updatepropsscript.py | 5 ++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index d7b679d9b3..efbb4b0c76 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -9,6 +9,8 @@ from itemchange import ItemChange from csvutil import records_to_csv import argparse +from langpropsutil import get_commit_for_language, LANG_FILENAME + def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit_2_id: str, show_commits: bool): """Determines the changes made in '.properties-MERGED' files from one commit to another commit. @@ -36,19 +38,36 @@ def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit def main(): parser = argparse.ArgumentParser(description="determines the updated, added, and deleted properties from the " + "'.properties-MERGED' files and generates a csv file containing " + - "the items changed.") - parser.add_argument(dest='repo_path', type=str, help='The path to the repo.') + "the items changed.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument(dest='output_path', type=str, help='The path to the output csv file.') - parser.add_argument(dest='commit_1_id', type=str, help='The commit for previous release.') - parser.add_argument('-lc', '--latest_commit', dest='commit_2_id', type=str, default='HEAD', required=False, + + parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, + help='The path to the repo. If not specified, path of script is used.') + parser.add_argument('-fc', '--first-commit', dest='commit_1_id', type=str, required=False, + help='The commit for previous release. This flag or the language flag need to be specified' + + ' in order to determine a start point for the difference.') + parser.add_argument('-lc', '--latest-commit', dest='commit_2_id', type=str, default='HEAD', required=False, help='The commit for current release.') - parser.add_argument('-nc', '--no_commits', dest='no_commits', action='store_true', default=False, + parser.add_argument('-nc', '--no-commits', dest='no_commits', action='store_true', default=False, required=False, help="Suppresses adding commits to the generated csv header.") + parser.add_argument('-l', '--language', dest='language', type=str, default='HEAD', required=False, + help='Specify the language in order to determine the first commit to use (i.e. \'ja\' for ' + + 'Japanese. This flag overrides the first-commit flag.') args = parser.parse_args() repo_path = args.repo_path output_path = args.output_path commit_1_id = args.commit_1_id + if args.language is not None: + commit_1_id = get_commit_for_language(args.language) + + if commit_1_id is None: + print('Either the first commit or language flag need to be specified. If specified, the language file, ' + + LANG_FILENAME + ', may not have the latest commit for the language.', file=sys.stderr) + parser.print_help(sys.stderr) + sys.exit(1) + commit_2_id = args.commit_2_id show_commits = not args.no_commits diff --git a/release_scripts/localization_scripts/updatepropsscript.py b/release_scripts/localization_scripts/updatepropsscript.py index f575ee847d..2b70fa00dd 100644 --- a/release_scripts/localization_scripts/updatepropsscript.py +++ b/release_scripts/localization_scripts/updatepropsscript.py @@ -194,6 +194,10 @@ def main(): parser.add_argument('-d', '--should_delete_idx', dest='should_delete_idx', action='store', type=int, default=None, required=False, help='The column index in the csv file providing whether or not the file ' + 'should be deleted. Any non-blank content will be treated as True.') + parser.add_argument('-c', '--commit_idx', dest='latest_commit_idx', action='store', type=int, default=3, + required=False, help='The column index in the csv file providing the commit for which this ' + + 'update applies. The commit should be located in the header row' + ) parser.add_argument('-f', '--file_rename', dest='file_rename', action='store', type=str, default=None, required=False, help='If specified, the properties file will be renamed to the argument' + ' preserving the specified relative path.') @@ -203,6 +207,7 @@ def main(): required=False, help="Whether or not to overwrite the previously existing properties files" + " ignoring previously existing values.") + args = parser.parse_args() repo_path = args.repo_path From 32b96584ce0efbccb2b801eb41cdb5c8450a07f3 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 13 Jul 2020 11:39:30 -0400 Subject: [PATCH 007/102] working through commit args --- release_scripts/localization_scripts/langpropsutil.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 release_scripts/localization_scripts/langpropsutil.py diff --git a/release_scripts/localization_scripts/langpropsutil.py b/release_scripts/localization_scripts/langpropsutil.py new file mode 100644 index 0000000000..3124137b9c --- /dev/null +++ b/release_scripts/localization_scripts/langpropsutil.py @@ -0,0 +1,7 @@ +LANG_FILENAME = 'lastupdated.properties' + +def get_commit_for_language(language: str) -> str: + pass + +def set_commit_for_language(language: str, latest_commit: str): + pass \ No newline at end of file From 8f9d24958575aa7bb99650755655f779162e11ea Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 13 Jul 2020 15:27:16 -0400 Subject: [PATCH 008/102] updates --- release_scripts/localization_scripts/updatepropsscript.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/release_scripts/localization_scripts/updatepropsscript.py b/release_scripts/localization_scripts/updatepropsscript.py index 2b70fa00dd..fec721d9f7 100644 --- a/release_scripts/localization_scripts/updatepropsscript.py +++ b/release_scripts/localization_scripts/updatepropsscript.py @@ -196,8 +196,7 @@ def main(): 'should be deleted. Any non-blank content will be treated as True.') parser.add_argument('-c', '--commit_idx', dest='latest_commit_idx', action='store', type=int, default=3, required=False, help='The column index in the csv file providing the commit for which this ' + - 'update applies. The commit should be located in the header row' - ) + 'update applies. The commit should be located in the header row. ') parser.add_argument('-f', '--file_rename', dest='file_rename', action='store', type=str, default=None, required=False, help='If specified, the properties file will be renamed to the argument' + ' preserving the specified relative path.') From f8cff3ff1defb8bd13e7462e313507349cfa18b1 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 13 Jul 2020 16:33:11 -0400 Subject: [PATCH 009/102] working through last updated items --- .../localization_scripts/allbundlesscript.py | 6 ++- .../localization_scripts/csvutil.py | 15 +++--- .../localization_scripts/diffscript.py | 3 +- .../localization_scripts/langpropsutil.py | 26 ++++++++-- .../localization_scripts/propsutil.py | 12 +++++ .../localization_scripts/updatepropsscript.py | 52 +++++++++++++------ 6 files changed, 85 insertions(+), 29 deletions(-) diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py index c4273a7a5d..2c73565939 100644 --- a/release_scripts/localization_scripts/allbundlesscript.py +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -7,6 +7,7 @@ repo is on correct branch (i.e. develop). import sys from gitutil import get_property_file_entries, get_commit_id from csvutil import records_to_csv +import pathlib import argparse @@ -33,13 +34,14 @@ def write_items_to_csv(repo_path: str, output_path: str, show_commit: bool): def main(): parser = argparse.ArgumentParser(description='Gathers all key-value pairs within .properties-MERGED files into ' + 'one csv file.') - parser.add_argument(dest='repo_path', type=str, help='The path to the repo.') parser.add_argument(dest='output_path', type=str, help='The path to the output csv file.') + parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, + help='The path to the repo. If not specified, path of script is used.') parser.add_argument('-nc', '--no_commit', dest='no_commit', action='store_true', default=False, required=False, help="Suppresses adding commits to the generated csv header.") args = parser.parse_args() - repo_path = args.repo_path + repo_path = args.repo_path if args.repo_path is not None else str(pathlib.Path(__file__).parent.absolute()) output_path = args.output_path show_commit = not args.no_commit diff --git a/release_scripts/localization_scripts/csvutil.py b/release_scripts/localization_scripts/csvutil.py index 5e0309998e..4349e781a6 100644 --- a/release_scripts/localization_scripts/csvutil.py +++ b/release_scripts/localization_scripts/csvutil.py @@ -1,7 +1,7 @@ """Provides tools for parsing and writing to a csv file. """ -from typing import List, Iterable, Iterator +from typing import List, Iterable, Tuple import csv import os @@ -27,21 +27,24 @@ def records_to_csv(output_path: str, rows: Iterable[List[str]]): writer.writerow(row) -def csv_to_records(input_path: str, header_row: bool) -> Iterator[List[str]]: +def csv_to_records(input_path: str, header_row: bool) -> Tuple[List[List[str]], List[str]]: """Writes rows to a csv file at the specified path. Args: input_path (str): The path where the csv file will be written. - header_row (bool): The rows to be written. Each row of a - list of strings will be written according - to their index (i.e. column 3 will be index 2). + header_row (bool): Whether or not there is a header row to be skipped. """ with open(input_path, encoding='utf-8') as csv_file: csv_reader = csv.reader(csv_file, delimiter=',') + header = None + results = [] for row in csv_reader: if header_row: + header = row header_row = False else: - yield row + results.append(row) + + return results, header diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index efbb4b0c76..3351c75abb 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -8,6 +8,7 @@ from gitutil import get_property_files_diff, get_commit_id from itemchange import ItemChange from csvutil import records_to_csv import argparse +import pathlib from langpropsutil import get_commit_for_language, LANG_FILENAME @@ -56,7 +57,7 @@ def main(): 'Japanese. This flag overrides the first-commit flag.') args = parser.parse_args() - repo_path = args.repo_path + repo_path = args.repo_path if args.repo_path is not None else str(pathlib.Path(__file__).parent.absolute()) output_path = args.output_path commit_1_id = args.commit_1_id if args.language is not None: diff --git a/release_scripts/localization_scripts/langpropsutil.py b/release_scripts/localization_scripts/langpropsutil.py index 3124137b9c..43e8707d1c 100644 --- a/release_scripts/localization_scripts/langpropsutil.py +++ b/release_scripts/localization_scripts/langpropsutil.py @@ -1,7 +1,27 @@ +from typing import Union +from propsutil import get_entry_dict +import pathlib +from os import path + + LANG_FILENAME = 'lastupdated.properties' -def get_commit_for_language(language: str) -> str: - pass +def get_last_update_key(language: str) -> str: + return "bundles.{lang}.lastupdated".format({lang=language}) + +def get_commit_for_language(language: str) -> Union[str, None]: + this_path = path.join(get_props_file_path(), LANG_FILENAME) + + if path.isfile(this_path): + lang_dict = get_entry_dict(this_path) + key = get_last_update_key(language) + if key in lang_dict: + return lang_dict[key] + + return None def set_commit_for_language(language: str, latest_commit: str): - pass \ No newline at end of file + pass + +def get_props_file_path() -> str: + return str(pathlib.Path(__file__).parent.absolute()) diff --git a/release_scripts/localization_scripts/propsutil.py b/release_scripts/localization_scripts/propsutil.py index b0b301c714..af48e8173c 100644 --- a/release_scripts/localization_scripts/propsutil.py +++ b/release_scripts/localization_scripts/propsutil.py @@ -10,6 +10,18 @@ from os import path DEFAULT_PROPS_EXTENSION = 'properties-MERGED' +def get_lang_bundle_name(language: str) -> str: + """ + Returns the bundle name for the specific language identifier provided. + Args: + language: The language identifier (i.e. 'ja' for Japanese) + + Returns: + The bundle name + """ + return 'Bundle_{lang}.properties'.format(lang=language) + + def get_entry_dict(file_contents: Union[str, IO]) -> Dict[str, str]: """Retrieves a dictionary mapping the properties represented in the string. diff --git a/release_scripts/localization_scripts/updatepropsscript.py b/release_scripts/localization_scripts/updatepropsscript.py index fec721d9f7..b975a5b115 100644 --- a/release_scripts/localization_scripts/updatepropsscript.py +++ b/release_scripts/localization_scripts/updatepropsscript.py @@ -5,10 +5,13 @@ This script requires the python libraries: jproperties. It also requires Python from typing import List, Dict, Tuple, Callable, Iterator import sys import os -from propsutil import set_entry_dict, get_entry_dict + +from langpropsutil import set_commit_for_language +from propsutil import set_entry_dict, get_entry_dict, get_lang_bundle_name from csvutil import csv_to_records from propentry import PropEntry import argparse +import pathlib def write_prop_entries(entries: Iterator[PropEntry], repo_path: str): @@ -184,32 +187,36 @@ def main(): parser = argparse.ArgumentParser(description='Updates properties files in the autopsy git repo.') parser.add_argument(dest='csv_file', type=str, help='The path to the csv file.') - parser.add_argument(dest='repo_path', type=str, help='The path to the repo.') - parser.add_argument('-p', '--path_idx', dest='path_idx', action='store', type=int, default=0, required=False, + + parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, + help='The path to the repo. If not specified, path of script is used.') + parser.add_argument('-p', '--path-idx', dest='path_idx', action='store', type=int, default=0, required=False, help='The column index in the csv file providing the relative path to the properties file.') - parser.add_argument('-k', '--key_idx', dest='key_idx', action='store', type=int, default=1, required=False, + parser.add_argument('-k', '--key-idx', dest='key_idx', action='store', type=int, default=1, required=False, help='The column index in the csv file providing the key within the properties file.') - parser.add_argument('-v', '--value_idx', dest='value_idx', action='store', type=int, default=2, required=False, + parser.add_argument('-v', '--value-idx', dest='value_idx', action='store', type=int, default=2, required=False, help='The column index in the csv file providing the value within the properties file.') - parser.add_argument('-d', '--should_delete_idx', dest='should_delete_idx', action='store', type=int, default=None, + parser.add_argument('-d', '--should-delete-idx', dest='should_delete_idx', action='store', type=int, default=None, required=False, help='The column index in the csv file providing whether or not the file ' + 'should be deleted. Any non-blank content will be treated as True.') - parser.add_argument('-c', '--commit_idx', dest='latest_commit_idx', action='store', type=int, default=3, + parser.add_argument('-c', '--commit-idx', dest='latest_commit_idx', action='store', type=int, default=3, required=False, help='The column index in the csv file providing the commit for which this ' + 'update applies. The commit should be located in the header row. ') - parser.add_argument('-f', '--file_rename', dest='file_rename', action='store', type=str, default=None, + parser.add_argument('-f', '--file-rename', dest='file_rename', action='store', type=str, default=None, required=False, help='If specified, the properties file will be renamed to the argument' + ' preserving the specified relative path.') - parser.add_argument('--has_no_header', dest='has_no_header', action='store_true', default=False, required=False, - help='Specify whether or not there is a header within the csv file.') - parser.add_argument('-o', '--should_overwrite', dest='should_overwrite', action='store_true', default=False, + parser.add_argument('-z', '--has-no-header', dest='has_no_header', action='store_true', default=False, + required=False, help='Specify whether or not there is a header within the csv file.') + parser.add_argument('-o', '--should-overwrite', dest='should_overwrite', action='store_true', default=False, required=False, help="Whether or not to overwrite the previously existing properties files" + " ignoring previously existing values.") - + parser.add_argument('-l', '--language', dest='language', type=str, default='HEAD', required=False, + help='Specify the language in order to update the last updated properties file and rename ' + + 'files within directories. This flag overrides the file-rename flag.') args = parser.parse_args() - repo_path = args.repo_path + repo_path = args.repo_path if args.repo_path is not None else str(pathlib.Path(__file__).parent.absolute()) input_path = args.csv_file path_idx = args.path_idx key_idx = args.key_idx @@ -217,26 +224,37 @@ def main(): has_header = not args.has_no_header overwrite = args.should_overwrite + # means of determining if a key should be deleted from a file if args.should_delete_idx is None: should_delete_converter = None else: def should_delete_converter(row_items: List[str]): return get_should_deleted(row_items, args.should_delete_idx) - if args.file_rename is None: - path_converter = None - else: + # provides the means of renaming the bundle file + if args.language is not None: + def path_converter(orig_path: str): + return get_new_rel_path(orig_path, get_lang_bundle_name(args.language)) + elif args.file_rename is not None: def path_converter(orig_path: str): return get_new_rel_path(orig_path, args.file_rename) + else: + path_converter = None - all_items = list(csv_to_records(input_path, has_header)) + # retrieve records from csv + all_items, header = list(csv_to_records(input_path, has_header)) prop_entries = get_prop_entries(all_items, path_idx, key_idx, value_idx, should_delete_converter, path_converter) + # write to files if overwrite: write_prop_entries(prop_entries, repo_path) else: update_prop_entries(prop_entries, repo_path) + # update the language last update if applicable + if args.language is not None and header is not None and len(header) > args.latest_commit_idx >= 0: + set_commit_for_language(args.language, header[args.latest_commit_idx]) + sys.exit(0) From f836176009e575824644a16b0b85afc16fdd98b3 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 14 Jul 2020 08:03:41 -0400 Subject: [PATCH 010/102] updates to include updated --- .../localization_scripts/allbundlesscript.py | 11 ++++-- .../localization_scripts/diffscript.py | 12 +++--- .../localization_scripts/envutil.py | 17 +++++++++ .../localization_scripts/gitutil.py | 3 ++ .../localization_scripts/langpropsutil.py | 38 +++++++++++-------- .../lastupdated.properties | 0 .../localization_scripts/propsutil.py | 17 +++++++++ .../localization_scripts/updatepropsscript.py | 29 +++++++------- 8 files changed, 90 insertions(+), 37 deletions(-) create mode 100644 release_scripts/localization_scripts/envutil.py create mode 100644 release_scripts/localization_scripts/lastupdated.properties diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py index 2c73565939..055cc835e8 100644 --- a/release_scripts/localization_scripts/allbundlesscript.py +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -5,9 +5,10 @@ repo is on correct branch (i.e. develop). """ import sys + +from envutil import get_proj_dir from gitutil import get_property_file_entries, get_commit_id from csvutil import records_to_csv -import pathlib import argparse @@ -25,6 +26,7 @@ def write_items_to_csv(repo_path: str, output_path: str, show_commit: bool): row_header.append(get_commit_id(repo_path, 'HEAD')) rows = [row_header] + for entry in get_property_file_entries(repo_path): rows.append([entry.rel_path, entry.key, entry.value]) @@ -32,8 +34,9 @@ def write_items_to_csv(repo_path: str, output_path: str, show_commit: bool): def main(): - parser = argparse.ArgumentParser(description='Gathers all key-value pairs within .properties-MERGED files into ' + - 'one csv file.') + parser = argparse.ArgumentParser(description='Gathers all key-value pairs within .properties-MERGED files into ' + 'one csv file.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument(dest='output_path', type=str, help='The path to the output csv file.') parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, help='The path to the repo. If not specified, path of script is used.') @@ -41,7 +44,7 @@ def main(): required=False, help="Suppresses adding commits to the generated csv header.") args = parser.parse_args() - repo_path = args.repo_path if args.repo_path is not None else str(pathlib.Path(__file__).parent.absolute()) + repo_path = args.repo_path if args.repo_path is not None else get_proj_dir() output_path = args.output_path show_commit = not args.no_commit diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index 3351c75abb..ebf50dc0c9 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -4,6 +4,8 @@ gitpython and jproperties. As a consequence, it also requires git >= 1.7.0 and """ import sys + +from envutil import get_proj_dir from gitutil import get_property_files_diff, get_commit_id from itemchange import ItemChange from csvutil import records_to_csv @@ -37,8 +39,8 @@ def write_diff_to_csv(repo_path: str, output_path: str, commit_1_id: str, commit def main(): - parser = argparse.ArgumentParser(description="determines the updated, added, and deleted properties from the " + - "'.properties-MERGED' files and generates a csv file containing " + + parser = argparse.ArgumentParser(description="determines the updated, added, and deleted properties from the " + "'.properties-MERGED' files and generates a csv file containing " "the items changed.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument(dest='output_path', type=str, help='The path to the output csv file.') @@ -46,18 +48,18 @@ def main(): parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, help='The path to the repo. If not specified, path of script is used.') parser.add_argument('-fc', '--first-commit', dest='commit_1_id', type=str, required=False, - help='The commit for previous release. This flag or the language flag need to be specified' + + help='The commit for previous release. This flag or the language flag need to be specified' ' in order to determine a start point for the difference.') parser.add_argument('-lc', '--latest-commit', dest='commit_2_id', type=str, default='HEAD', required=False, help='The commit for current release.') parser.add_argument('-nc', '--no-commits', dest='no_commits', action='store_true', default=False, required=False, help="Suppresses adding commits to the generated csv header.") parser.add_argument('-l', '--language', dest='language', type=str, default='HEAD', required=False, - help='Specify the language in order to determine the first commit to use (i.e. \'ja\' for ' + + help='Specify the language in order to determine the first commit to use (i.e. \'ja\' for ' 'Japanese. This flag overrides the first-commit flag.') args = parser.parse_args() - repo_path = args.repo_path if args.repo_path is not None else str(pathlib.Path(__file__).parent.absolute()) + repo_path = args.repo_path if args.repo_path is not None else get_proj_dir() output_path = args.output_path commit_1_id = args.commit_1_id if args.language is not None: diff --git a/release_scripts/localization_scripts/envutil.py b/release_scripts/localization_scripts/envutil.py new file mode 100644 index 0000000000..cec2a00eda --- /dev/null +++ b/release_scripts/localization_scripts/envutil.py @@ -0,0 +1,17 @@ +"""Functions relating to the project environment. +""" + +import pathlib +from typing import Union + + +def get_proj_dir(path: Union[pathlib.PurePath, str] = __file__) -> str: + """ + Gets parent directory of this file (and subsequently, the project). + Args: + path: Can be overridden to provide a different file. This will return the parent of that file in that instance. + + Returns: + The project folder or the parent folder of the file provided. + """ + return str(pathlib.Path(path).parent.absolute()) diff --git a/release_scripts/localization_scripts/gitutil.py b/release_scripts/localization_scripts/gitutil.py index 65db268fc8..9b9667d421 100644 --- a/release_scripts/localization_scripts/gitutil.py +++ b/release_scripts/localization_scripts/gitutil.py @@ -1,3 +1,6 @@ +"""Functions relating to using git and GitPython with an existing repo. +""" + from git import Repo, Diff, Blob from typing import List, Union, Iterator, Tuple, Any from itemchange import ItemChange, get_changed diff --git a/release_scripts/localization_scripts/langpropsutil.py b/release_scripts/localization_scripts/langpropsutil.py index 43e8707d1c..1104e14665 100644 --- a/release_scripts/localization_scripts/langpropsutil.py +++ b/release_scripts/localization_scripts/langpropsutil.py @@ -1,27 +1,35 @@ +"""Functions handling retrieving and storing when a language was last updated. +""" + from typing import Union -from propsutil import get_entry_dict -import pathlib +from envutil import get_proj_dir +from propsutil import get_entry_dict_from_path, update_entry_dict from os import path LANG_FILENAME = 'lastupdated.properties' -def get_last_update_key(language: str) -> str: - return "bundles.{lang}.lastupdated".format({lang=language}) + +def _get_last_update_key(language: str) -> str: + return "bundles.{lang}.lastupdated".format(lang=language) + + +def _get_props_path(): + return path.join(get_proj_dir(), LANG_FILENAME) + def get_commit_for_language(language: str) -> Union[str, None]: - this_path = path.join(get_props_file_path(), LANG_FILENAME) + lang_dict = get_entry_dict_from_path(_get_props_path()) + if lang_dict is None: + return None - if path.isfile(this_path): - lang_dict = get_entry_dict(this_path) - key = get_last_update_key(language) - if key in lang_dict: - return lang_dict[key] + key = _get_last_update_key(language) + if key not in lang_dict: + return None + + return lang_dict[key] - return None def set_commit_for_language(language: str, latest_commit: str): - pass - -def get_props_file_path() -> str: - return str(pathlib.Path(__file__).parent.absolute()) + key = _get_last_update_key(language) + update_entry_dict({key: latest_commit}, _get_props_path()) diff --git a/release_scripts/localization_scripts/lastupdated.properties b/release_scripts/localization_scripts/lastupdated.properties new file mode 100644 index 0000000000..e69de29bb2 diff --git a/release_scripts/localization_scripts/propsutil.py b/release_scripts/localization_scripts/propsutil.py index af48e8173c..dfb5b0e323 100644 --- a/release_scripts/localization_scripts/propsutil.py +++ b/release_scripts/localization_scripts/propsutil.py @@ -37,6 +37,23 @@ def get_entry_dict(file_contents: Union[str, IO]) -> Dict[str, str]: return props.properties +def get_entry_dict_from_path(props_path: str) -> Union[Dict[str, str], None]: + """ + Retrieves a dictionary mapping the properties represented in the string or None if no properties file can be found + at that path. + Args: + props_path: The path to the properties file. + + Returns: The entry dictionary for that properties file. + + """ + if os.path.isfile(props_path): + with open(props_path, "rb") as f: + return get_entry_dict(f) + else: + return None + + def set_entry_dict(contents: Dict[str, str], file_path: str): """Sets the property file to the key-value pairs of the contents dictionary. diff --git a/release_scripts/localization_scripts/updatepropsscript.py b/release_scripts/localization_scripts/updatepropsscript.py index b975a5b115..a1d7bb40f5 100644 --- a/release_scripts/localization_scripts/updatepropsscript.py +++ b/release_scripts/localization_scripts/updatepropsscript.py @@ -6,12 +6,12 @@ from typing import List, Dict, Tuple, Callable, Iterator import sys import os +from envutil import get_proj_dir from langpropsutil import set_commit_for_language from propsutil import set_entry_dict, get_entry_dict, get_lang_bundle_name from csvutil import csv_to_records from propentry import PropEntry import argparse -import pathlib def write_prop_entries(entries: Iterator[PropEntry], repo_path: str): @@ -42,10 +42,8 @@ def update_prop_entries(entries: Iterator[PropEntry], repo_path: str): for rel_path, (entries, to_delete) in items_by_file.items(): abs_path = os.path.join(repo_path, rel_path) - if os.path.isfile(abs_path): - with open(abs_path, "rb") as f: - prop_items = get_entry_dict(f) - else: + prop_items = get_entry_dict(abs_path) + if prop_items is None: prop_items = {} for key_to_delete in to_delete: @@ -184,9 +182,14 @@ def get_new_rel_path(orig_path: str, new_filename: str) -> str: def main(): - parser = argparse.ArgumentParser(description='Updates properties files in the autopsy git repo.') + parser = argparse.ArgumentParser(description='Updates properties files in the autopsy git repo.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument(dest='csv_file', type=str, help='The path to the csv file.') + parser.add_argument(dest='csv_file', type=str, help='The path to the csv file. The default format for the csv ' + 'file has columns of relative path, properties file key, ' + 'properties file value, and commit id for how recent these ' + 'updates are. A header row is expected by default and the ' + 'commit id, if specified, should only be in the first row.') parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, help='The path to the repo. If not specified, path of script is used.') @@ -197,26 +200,26 @@ def main(): parser.add_argument('-v', '--value-idx', dest='value_idx', action='store', type=int, default=2, required=False, help='The column index in the csv file providing the value within the properties file.') parser.add_argument('-d', '--should-delete-idx', dest='should_delete_idx', action='store', type=int, default=None, - required=False, help='The column index in the csv file providing whether or not the file ' + + required=False, help='The column index in the csv file providing whether or not the file ' 'should be deleted. Any non-blank content will be treated as True.') parser.add_argument('-c', '--commit-idx', dest='latest_commit_idx', action='store', type=int, default=3, - required=False, help='The column index in the csv file providing the commit for which this ' + + required=False, help='The column index in the csv file providing the commit for which this ' 'update applies. The commit should be located in the header row. ') parser.add_argument('-f', '--file-rename', dest='file_rename', action='store', type=str, default=None, - required=False, help='If specified, the properties file will be renamed to the argument' + + required=False, help='If specified, the properties file will be renamed to the argument' ' preserving the specified relative path.') parser.add_argument('-z', '--has-no-header', dest='has_no_header', action='store_true', default=False, required=False, help='Specify whether or not there is a header within the csv file.') parser.add_argument('-o', '--should-overwrite', dest='should_overwrite', action='store_true', default=False, - required=False, help="Whether or not to overwrite the previously existing properties files" + + required=False, help="Whether or not to overwrite the previously existing properties files" " ignoring previously existing values.") parser.add_argument('-l', '--language', dest='language', type=str, default='HEAD', required=False, - help='Specify the language in order to update the last updated properties file and rename ' + + help='Specify the language in order to update the last updated properties file and rename ' 'files within directories. This flag overrides the file-rename flag.') args = parser.parse_args() - repo_path = args.repo_path if args.repo_path is not None else str(pathlib.Path(__file__).parent.absolute()) + repo_path = args.repo_path if args.repo_path is not None else get_proj_dir() input_path = args.csv_file path_idx = args.path_idx key_idx = args.key_idx From d8921d2ac4bfbf75a7a2b633f550628210e12b8b Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 14 Jul 2020 08:53:47 -0400 Subject: [PATCH 011/102] add in deletion row expection for update and a readme --- release_scripts/localization_scripts/README.md | 17 +++++++++++++++++ .../localization_scripts/updatepropsscript.py | 16 ++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 release_scripts/localization_scripts/README.md diff --git a/release_scripts/localization_scripts/README.md b/release_scripts/localization_scripts/README.md new file mode 100644 index 0000000000..f2db66fcc8 --- /dev/null +++ b/release_scripts/localization_scripts/README.md @@ -0,0 +1,17 @@ +## Description + +This folder provides tools to handle updates of bundle files for language localization. There are three main scripts: +- `allbundlesscript.py` - generates a csv file containing the relative path of the bundle file, the key, and the value for each property. +- `diffscript.py` - determines the property values that have changed between two commits and generates a csv file containing the relative path, the key, the previous value, the new value, and the change type (addition, deletion, change). +- `updatepropsscript.py` - Given a csv file containing the relative path of the bundle, the key, and the new value, will update the property values for a given language within the project. + +All of these scripts provide more details on usage by calling the script with `-h`. + +## Basic Localization Update Workflow + +1. Call `python3 diffscript.py -l ` to generate a csv file containing differences in properties file values from the language's previous commit to the `HEAD` commit. The language identifier should be the abbreviated identifier used for the bundle (i.e. 'ja' for Japanese). +2. Update csv file with translations +3. Call `python3 updatepropsscript.py -l ` to update properties files based on the newly generated csv file. The csv file should be formatted such that the columns are bundle relative path, property files key, translated value and commit id for the latest commit id for which these changes represent. The commit id only needs to be in the header row. + +## Localization Generation for the First Time +First-time updates should follow a similar procedure except that instead of calling `diffscript.py`, call `python3 allbundlesscript ` to generate a csv file with relative paths of bundle files, property file keys, property file values. \ No newline at end of file diff --git a/release_scripts/localization_scripts/updatepropsscript.py b/release_scripts/localization_scripts/updatepropsscript.py index a1d7bb40f5..289478573c 100644 --- a/release_scripts/localization_scripts/updatepropsscript.py +++ b/release_scripts/localization_scripts/updatepropsscript.py @@ -150,7 +150,7 @@ def get_prop_entries(rows: List[List[str]], def get_should_deleted(row_items: List[str], requested_idx: int) -> bool: - """If there is a value at row_items[requested_idx] and that value is not empty, then this will return true. + """If there is a value at row_items[requested_idx] and that value starts with 'DELET', then this will return true. Args: row_items (List[str]): The row items. @@ -159,7 +159,7 @@ def get_should_deleted(row_items: List[str], requested_idx: int) -> bool: Returns: bool: True if the row specifies it should be deleted. """ - if idx_bounded(requested_idx, len(row_items)) and len((row_items[requested_idx].strip())) > 0: + if idx_bounded(requested_idx, len(row_items)) and row_items[requested_idx].strip().upper().startswith('DELET'): return True else: return False @@ -187,8 +187,10 @@ def main(): parser.add_argument(dest='csv_file', type=str, help='The path to the csv file. The default format for the csv ' 'file has columns of relative path, properties file key, ' - 'properties file value, and commit id for how recent these ' - 'updates are. A header row is expected by default and the ' + 'properties file value, whether or not the key should be ' + 'deleted, and commit id for how recent these updates are. ' + 'If the key should be deleted, the deletion row should be ' + '\'DELETION.\' A header row is expected by default and the ' 'commit id, if specified, should only be in the first row.') parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, @@ -199,12 +201,13 @@ def main(): help='The column index in the csv file providing the key within the properties file.') parser.add_argument('-v', '--value-idx', dest='value_idx', action='store', type=int, default=2, required=False, help='The column index in the csv file providing the value within the properties file.') - parser.add_argument('-d', '--should-delete-idx', dest='should_delete_idx', action='store', type=int, default=None, + parser.add_argument('-d', '--should-delete-idx', dest='should_delete_idx', action='store', type=int, default=3, required=False, help='The column index in the csv file providing whether or not the file ' 'should be deleted. Any non-blank content will be treated as True.') - parser.add_argument('-c', '--commit-idx', dest='latest_commit_idx', action='store', type=int, default=3, + parser.add_argument('-c', '--commit-idx', dest='latest_commit_idx', action='store', type=int, default=4, required=False, help='The column index in the csv file providing the commit for which this ' 'update applies. The commit should be located in the header row. ') + parser.add_argument('-f', '--file-rename', dest='file_rename', action='store', type=str, default=None, required=False, help='If specified, the properties file will be renamed to the argument' ' preserving the specified relative path.') @@ -213,6 +216,7 @@ def main(): parser.add_argument('-o', '--should-overwrite', dest='should_overwrite', action='store_true', default=False, required=False, help="Whether or not to overwrite the previously existing properties files" " ignoring previously existing values.") + parser.add_argument('-l', '--language', dest='language', type=str, default='HEAD', required=False, help='Specify the language in order to update the last updated properties file and rename ' 'files within directories. This flag overrides the file-rename flag.') From f4a73dcc65ec5c71f520bc3f1f6f8441d59b2f58 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 14 Jul 2020 09:33:32 -0400 Subject: [PATCH 012/102] error handling and repo path determining --- .../localization_scripts/allbundlesscript.py | 4 ++-- .../localization_scripts/csvutil.py | 15 +++++++++------ .../localization_scripts/diffscript.py | 4 ++-- .../localization_scripts/gitutil.py | 15 +++++++++++++++ .../localization_scripts/propsutil.py | 18 +++++++++++------- .../localization_scripts/updatepropsscript.py | 9 +++++---- 6 files changed, 44 insertions(+), 21 deletions(-) diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py index 055cc835e8..b9efd7b0b3 100644 --- a/release_scripts/localization_scripts/allbundlesscript.py +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -7,7 +7,7 @@ repo is on correct branch (i.e. develop). import sys from envutil import get_proj_dir -from gitutil import get_property_file_entries, get_commit_id +from gitutil import get_property_file_entries, get_commit_id, get_git_root from csvutil import records_to_csv import argparse @@ -44,7 +44,7 @@ def main(): required=False, help="Suppresses adding commits to the generated csv header.") args = parser.parse_args() - repo_path = args.repo_path if args.repo_path is not None else get_proj_dir() + repo_path = args.repo_path if args.repo_path is not None else get_git_root(get_proj_dir()) output_path = args.output_path show_commit = not args.no_commit diff --git a/release_scripts/localization_scripts/csvutil.py b/release_scripts/localization_scripts/csvutil.py index 4349e781a6..aa382944e1 100644 --- a/release_scripts/localization_scripts/csvutil.py +++ b/release_scripts/localization_scripts/csvutil.py @@ -40,11 +40,14 @@ def csv_to_records(input_path: str, header_row: bool) -> Tuple[List[List[str]], header = None results = [] - for row in csv_reader: - if header_row: - header = row - header_row = False - else: - results.append(row) + try: + for row in csv_reader: + if header_row: + header = row + header_row = False + else: + results.append(row) + except Exception as e: + raise Exception("There was an error parsing csv {path}".format(path=input_path), e) return results, header diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index ebf50dc0c9..b8dfb522c2 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -6,7 +6,7 @@ gitpython and jproperties. As a consequence, it also requires git >= 1.7.0 and import sys from envutil import get_proj_dir -from gitutil import get_property_files_diff, get_commit_id +from gitutil import get_property_files_diff, get_commit_id, get_git_root from itemchange import ItemChange from csvutil import records_to_csv import argparse @@ -59,7 +59,7 @@ def main(): 'Japanese. This flag overrides the first-commit flag.') args = parser.parse_args() - repo_path = args.repo_path if args.repo_path is not None else get_proj_dir() + repo_path = args.repo_path if args.repo_path is not None else get_git_root(get_proj_dir()) output_path = args.output_path commit_1_id = args.commit_1_id if args.language is not None: diff --git a/release_scripts/localization_scripts/gitutil.py b/release_scripts/localization_scripts/gitutil.py index 9b9667d421..333bee1440 100644 --- a/release_scripts/localization_scripts/gitutil.py +++ b/release_scripts/localization_scripts/gitutil.py @@ -13,6 +13,21 @@ def get_text(blob: Blob) -> str: return blob.data_stream.read().decode('utf-8') +def get_git_root(child_path: str) -> str: + """ + Taken from https://stackoverflow.com/questions/22081209/find-the-root-of-the-git-repository-where-the-file-lives, + this obtains the root path of the git repo in which this file exists. + Args: + child_path: The path of a child within the repo. + + Returns: The repo root path. + + """ + git_repo = Repo(child_path, search_parent_directories=True) + git_root = git_repo.git.rev_parse("--show-toplevel") + return git_root + + def get_changed_from_diff(rel_path: str, diff: Diff) -> List[ItemChange]: """Determines changes from a git python diff. diff --git a/release_scripts/localization_scripts/propsutil.py b/release_scripts/localization_scripts/propsutil.py index dfb5b0e323..6d0fd7a5b5 100644 --- a/release_scripts/localization_scripts/propsutil.py +++ b/release_scripts/localization_scripts/propsutil.py @@ -33,7 +33,10 @@ def get_entry_dict(file_contents: Union[str, IO]) -> Dict[str, str]: """ props = Properties() - props.load(file_contents, "utf-8") + try: + props.load(file_contents) + except Exception as e: + raise Exception("There was an error loading properties file {file}".format(file=file_contents), e) return props.properties @@ -84,11 +87,12 @@ def update_entry_dict(contents: Dict[str, str], file_path: str): """ contents_to_edit = contents.copy() - if path.isfile(file_path): - cur_dict = get_entry_dict(file_path) - for cur_key, cur_val in cur_dict.values(): - # only update contents if contents does not already have key - if cur_key not in contents_to_edit: - contents_to_edit[cur_key] = cur_val + cur_dict = get_entry_dict_from_path(file_path) + if cur_dict is None: + cur_dict = {} + for cur_key, cur_val in cur_dict.items(): + # only update contents if contents does not already have key + if cur_key not in contents_to_edit: + contents_to_edit[cur_key] = cur_val set_entry_dict(contents_to_edit, file_path) diff --git a/release_scripts/localization_scripts/updatepropsscript.py b/release_scripts/localization_scripts/updatepropsscript.py index 289478573c..99723a395d 100644 --- a/release_scripts/localization_scripts/updatepropsscript.py +++ b/release_scripts/localization_scripts/updatepropsscript.py @@ -7,8 +7,9 @@ import sys import os from envutil import get_proj_dir +from gitutil import get_git_root from langpropsutil import set_commit_for_language -from propsutil import set_entry_dict, get_entry_dict, get_lang_bundle_name +from propsutil import set_entry_dict, get_entry_dict_from_path, get_lang_bundle_name from csvutil import csv_to_records from propentry import PropEntry import argparse @@ -42,7 +43,7 @@ def update_prop_entries(entries: Iterator[PropEntry], repo_path: str): for rel_path, (entries, to_delete) in items_by_file.items(): abs_path = os.path.join(repo_path, rel_path) - prop_items = get_entry_dict(abs_path) + prop_items = get_entry_dict_from_path(abs_path) if prop_items is None: prop_items = {} @@ -194,7 +195,7 @@ def main(): 'commit id, if specified, should only be in the first row.') parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, - help='The path to the repo. If not specified, path of script is used.') + help='The path to the repo. If not specified, parent repo of path of script is used.') parser.add_argument('-p', '--path-idx', dest='path_idx', action='store', type=int, default=0, required=False, help='The column index in the csv file providing the relative path to the properties file.') parser.add_argument('-k', '--key-idx', dest='key_idx', action='store', type=int, default=1, required=False, @@ -223,7 +224,7 @@ def main(): args = parser.parse_args() - repo_path = args.repo_path if args.repo_path is not None else get_proj_dir() + repo_path = args.repo_path if args.repo_path is not None else get_git_root(get_proj_dir()) input_path = args.csv_file path_idx = args.path_idx key_idx = args.key_idx From 2ef53e6a302ae528f7d1773a81c199917acd00e9 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 14 Jul 2020 11:05:36 -0400 Subject: [PATCH 013/102] Updated autopsy to use the new CaseUcoExporter tool --- .../caseuco/CaseUcoReportGenerator.java | 466 ------------- .../modules/caseuco/CaseUcoReportModule.java | 186 ++++-- .../PortableCaseReportModule.java | 614 +++++++++--------- 3 files changed, 435 insertions(+), 831 deletions(-) delete mode 100755 Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportGenerator.java diff --git a/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportGenerator.java b/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportGenerator.java deleted file mode 100755 index 9cc4467835..0000000000 --- a/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportGenerator.java +++ /dev/null @@ -1,466 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2018-2020 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.report.modules.caseuco; - -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.io.IOException; -import java.nio.file.Path; -import java.util.SimpleTimeZone; -import java.util.TimeZone; -import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.Case.CaseType; -import org.sleuthkit.autopsy.datamodel.ContentUtils; -import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.SleuthkitCase; -import com.fasterxml.jackson.core.JsonEncoding; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.util.DefaultIndenter; -import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Strings; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import org.sleuthkit.datamodel.Content; -import org.sleuthkit.datamodel.Image; -import org.sleuthkit.datamodel.TskCoreException; - -/** - * Writes Autopsy DataModel objects to Case UCO format. - * - * Clients are expected to add the Case first. Then they should add each data - * source before adding any files for that data source. - * - * Here is an example, where we add everything: - * - * Path directory = Paths.get("C:", "Reports"); - * CaseUcoReportGenerator caseUco = new CaseUcoReportGenerator(directory, "my-report"); - * - * Case caseObj = Case.getCurrentCase(); - * caseUco.addCase(caseObj); - * List dataSources = caseObj.getDataSources(); - * for(Content dataSource : dataSources) { - * caseUco.addDataSource(dataSource, caseObj); - * List files = getAllFilesInDataSource(dataSource); - * for(AbstractFile file : files) { - * caseUco.addFile(file, dataSource); - * } - * } - * - * Path reportOutput = caseUco.generateReport(); - * //Done. Report at - "C:\Reports\my-report.json-ld" - * - * Please note that the life cycle for this class ends with generateReport(). - * The underlying file handle to 'my-report.json-ld' will be closed. Any further - * calls to addX() will result in an IOException. - */ -public final class CaseUcoReportGenerator { - - private static final String EXTENSION = "json-ld"; - - private final TimeZone timeZone; - private final Path reportPath; - private final JsonGenerator reportGenerator; - - /** - * Creates a CaseUCO Report Generator that writes a report in the specified - * directory. - * - * TimeZone is assumed to be GMT+0 for formatting file creation time, - * accessed time and modified time. - * - * @param directory Directory to write the CaseUCO report file. Assumes the - * calling thread has write access to the directory and that the directory - * exists. - * @param reportName Name of the CaseUCO report file. - * @throws IOException If an I/O error occurs - */ - public CaseUcoReportGenerator(Path directory, String reportName) throws IOException { - this.reportPath = directory.resolve(reportName + "." + EXTENSION); - - JsonFactory jsonGeneratorFactory = new JsonFactory(); - reportGenerator = jsonGeneratorFactory.createGenerator(reportPath.toFile(), JsonEncoding.UTF8); - // Puts a newline between each Key, Value pair for readability. - reportGenerator.setPrettyPrinter(new DefaultPrettyPrinter() - .withObjectIndenter(new DefaultIndenter(" ", "\n"))); - - ObjectMapper mapper = new ObjectMapper(); - mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); - mapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); - - reportGenerator.setCodec(mapper); - - reportGenerator.writeStartObject(); - reportGenerator.writeFieldName("@graph"); - reportGenerator.writeStartArray(); - - //Assume GMT+0 - this.timeZone = new SimpleTimeZone(0, "GMT"); - } - - /** - * Adds an AbstractFile instance to the Case UCO report. - * - * @param file AbstractFile instance to write - * @param parentDataSource The parent data source for this abstract file. It - * is assumed that this parent has been written to the report (via - * addDataSource) prior to this call. Otherwise, the report may be invalid. - * @throws IOException If an I/O error occurs. - * @throws TskCoreException - */ - public void addFile(AbstractFile file, Content parentDataSource) throws IOException, TskCoreException { - addFile(file, parentDataSource, null); - } - - /** - * Adds an AbstractFile instance to the Case UCO report. - * - * @param file AbstractFile instance to write - * @param parentDataSource The parent data source for this abstract file. It - * is assumed that this parent has been written to the report (via - * addDataSource) prior to this call. Otherwise, the report may be invalid. - * @param localPath The location of the file on secondary storage, somewhere - * other than the case. Example: local disk. This value will be ignored if - * it is null. - * @throws IOException - * @throws TskCoreException - */ - public void addFile(AbstractFile file, Content parentDataSource, Path localPath) throws IOException, TskCoreException { - String fileTraceId = getFileTraceId(file); - - //Create the Trace CASE node, which will contain attributes about some evidence. - //Trace is the standard term for evidence. For us, this means file system files. - CASENode fileTrace = new CASENode(fileTraceId, "Trace"); - - //The bits of evidence for each Trace node are contained within Property - //Bundles. There are a number of Property Bundles available in the CASE ontology. - - //Build up the File Property Bundle, as the name implies - properties of - //the file itself. - CASEPropertyBundle filePropertyBundle = createFileBundle(file); - fileTrace.addBundle(filePropertyBundle); - - //Build up the ContentData Property Bundle, as the name implies - properties of - //the File data itself. - CASEPropertyBundle contentDataPropertyBundle = createContentDataBundle(file); - fileTrace.addBundle(contentDataPropertyBundle); - - if(localPath != null) { - String urlTraceId = getURLTraceId(file); - CASENode urlTrace = new CASENode(urlTraceId, "Trace"); - CASEPropertyBundle urlPropertyBundle = new CASEPropertyBundle("URL"); - urlPropertyBundle.addProperty("fullValue", localPath.toString()); - urlTrace.addBundle(urlPropertyBundle); - - contentDataPropertyBundle.addProperty("dataPayloadReferenceUrl", urlTraceId); - reportGenerator.writeObject(urlTrace); - } - - //Create the Relationship CASE node. This defines how the Trace CASE node described above - //is related to another CASE node (in this case, the parent data source). - String relationshipID = getRelationshipId(file); - CASENode relationship = createRelationshipNode(relationshipID, - fileTraceId, getDataSourceTraceId(parentDataSource)); - - //Build up the PathRelation bundle for the relationship node, - //as the name implies - the Path of the Trace in the data source. - CASEPropertyBundle pathRelationPropertyBundle = new CASEPropertyBundle("PathRelation"); - pathRelationPropertyBundle.addProperty("path", file.getUniquePath()); - relationship.addBundle(pathRelationPropertyBundle); - - //This completes the triage, write them to JSON. - reportGenerator.writeObject(fileTrace); - reportGenerator.writeObject(relationship); - } - - private String getURLTraceId(Content content) { - return "url-" + content.getId(); - } - - /** - * All relationship nodes will be the same within our context. Namely, contained-within - * and isDirectional as true. - */ - private CASENode createRelationshipNode(String relationshipID, String sourceID, String targetID) { - CASENode relationship = new CASENode(relationshipID, "Relationship"); - relationship.addProperty("source", sourceID); - relationship.addProperty("target", targetID); - relationship.addProperty("kindOfRelationship", "contained-within"); - relationship.addProperty("isDirectional", true); - return relationship; - } - - /** - * Creates a File Property Bundle with a selection of file attributes. - */ - private CASEPropertyBundle createFileBundle(AbstractFile file) throws TskCoreException { - CASEPropertyBundle filePropertyBundle = new CASEPropertyBundle("File"); - String createdTime = ContentUtils.getStringTimeISO8601(file.getCrtime(), timeZone); - String accessedTime = ContentUtils.getStringTimeISO8601(file.getAtime(), timeZone); - String modifiedTime = ContentUtils.getStringTimeISO8601(file.getMtime(), timeZone); - filePropertyBundle.addProperty("createdTime", createdTime); - filePropertyBundle.addProperty("accessedTime", accessedTime); - filePropertyBundle.addProperty("modifiedTime", modifiedTime); - if (!Strings.isNullOrEmpty(file.getNameExtension())) { - filePropertyBundle.addProperty("extension", file.getNameExtension()); - } - filePropertyBundle.addProperty("fileName", file.getName()); - filePropertyBundle.addProperty("filePath", file.getUniquePath()); - filePropertyBundle.addProperty("isDirectory", file.isDir()); - filePropertyBundle.addProperty("sizeInBytes", Long.toString(file.getSize())); - return filePropertyBundle; - } - - /** - * Creates a Content Data Property Bundle with a selection of file attributes. - */ - private CASEPropertyBundle createContentDataBundle(AbstractFile file) { - CASEPropertyBundle contentDataPropertyBundle = new CASEPropertyBundle("ContentData"); - if (!Strings.isNullOrEmpty(file.getMIMEType())) { - contentDataPropertyBundle.addProperty("mimeType", file.getMIMEType()); - } - if (!Strings.isNullOrEmpty(file.getMd5Hash())) { - List hashPropertyBundles = new ArrayList<>(); - CASEPropertyBundle md5HashPropertyBundle = new CASEPropertyBundle("Hash"); - md5HashPropertyBundle.addProperty("hashMethod", "MD5"); - md5HashPropertyBundle.addProperty("hashValue", file.getMd5Hash()); - hashPropertyBundles.add(md5HashPropertyBundle); - contentDataPropertyBundle.addProperty("hash", hashPropertyBundles); - } - contentDataPropertyBundle.addProperty("sizeInBytes", Long.toString(file.getSize())); - return contentDataPropertyBundle; - } - - /** - * Creates a unique CASE Node file trace id. - */ - private String getFileTraceId(AbstractFile file) { - return "file-" + file.getId(); - } - - /** - * Creates a unique CASE Node relationship id value. - */ - private String getRelationshipId(Content content) { - return "relationship-" + content.getId(); - } - - /** - * Adds a Content instance (which is known to be a DataSource) to the CASE - * report. This means writing a selection of attributes to a CASE or UCO - * object. - * - * @param dataSource Datasource content to write - * @param parentCase The parent case that this data source belongs in. It is - * assumed that this parent has been written to the report (via addCase) - * prior to this call. Otherwise, the report may be invalid. - */ - public void addDataSource(Content dataSource, Case parentCase) throws IOException, TskCoreException { - String dataSourceTraceId = this.getDataSourceTraceId(dataSource); - - CASENode dataSourceTrace = new CASENode(dataSourceTraceId, "Trace"); - CASEPropertyBundle filePropertyBundle = new CASEPropertyBundle("File"); - - String dataSourcePath = getDataSourcePath(dataSource); - - filePropertyBundle.addProperty("filePath", dataSourcePath); - dataSourceTrace.addBundle(filePropertyBundle); - - if (dataSource.getSize() > 0) { - CASEPropertyBundle contentDataPropertyBundle = new CASEPropertyBundle("ContentData"); - contentDataPropertyBundle.addProperty("sizeInBytes", Long.toString(dataSource.getSize())); - dataSourceTrace.addBundle(contentDataPropertyBundle); - } - - // create a "relationship" entry between the case and the data source - String caseTraceId = getCaseTraceId(parentCase); - String relationshipTraceId = getRelationshipId(dataSource); - CASENode relationship = createRelationshipNode(relationshipTraceId, - dataSourceTraceId, caseTraceId); - - CASEPropertyBundle pathRelationBundle = new CASEPropertyBundle("PathRelation"); - pathRelationBundle.addProperty("path", dataSourcePath); - relationship.addBundle(pathRelationBundle); - - //This completes the triage, write them to JSON. - reportGenerator.writeObject(dataSourceTrace); - reportGenerator.writeObject(relationship); - } - - private String getDataSourcePath(Content dataSource) { - String dataSourcePath = ""; - if (dataSource instanceof Image) { - String[] paths = ((Image) dataSource).getPaths(); - if (paths.length > 0) { - //Get the first data source in the path, as this will - //be reflected in each file's uniquePath. - dataSourcePath = paths[0]; - } - } else { - dataSourcePath = dataSource.getName(); - } - dataSourcePath = dataSourcePath.replaceAll("\\\\", "/"); - return dataSourcePath; - } - - /** - * Creates a unique Case UCO trace id for a data source. - * - * @param dataSource - * @return - */ - private String getDataSourceTraceId(Content dataSource) { - return "data-source-" + dataSource.getId(); - } - - /** - * Adds a Case instance to the Case UCO report. This means writing a - * selection of Case attributes to a CASE/UCO object. - * - * @param caseObj Case instance to include in the report. - * @throws IOException If an I/O error is encountered. - */ - public void addCase(Case caseObj) throws IOException { - SleuthkitCase skCase = caseObj.getSleuthkitCase(); - - String caseDirPath = skCase.getDbDirPath(); - String caseTraceId = getCaseTraceId(caseObj); - CASENode caseTrace = new CASENode(caseTraceId, "Trace"); - CASEPropertyBundle filePropertyBundle = new CASEPropertyBundle("File"); - - // replace double slashes with single ones - caseDirPath = caseDirPath.replaceAll("\\\\", "/"); - - Case.CaseType caseType = caseObj.getCaseType(); - if (caseType.equals(CaseType.SINGLE_USER_CASE)) { - filePropertyBundle.addProperty("filePath", caseDirPath + "/" + skCase.getDatabaseName()); - filePropertyBundle.addProperty("isDirectory", false); - } else { - filePropertyBundle.addProperty("filePath", caseDirPath); - filePropertyBundle.addProperty("isDirectory", true); - } - - caseTrace.addBundle(filePropertyBundle); - reportGenerator.writeObject(caseTrace); - } - - /** - * Creates a unique Case UCO trace id for a Case. - * - * @param caseObj - * @return - */ - private String getCaseTraceId(Case caseObj) { - return "case-" + caseObj.getName(); - } - - /** - * Returns a Path to the completed Case UCO report file. - * - * This marks the end of the CaseUcoReportGenerator's life cycle. This - * function will close an underlying file handles, meaning any subsequent - * calls to addX() will result in an IOException. - * - * @return The Path to the finalized report. - * @throws IOException If an I/O error occurs. - */ - public Path generateReport() throws IOException { - //Finalize the report. - reportGenerator.writeEndArray(); - reportGenerator.writeEndObject(); - reportGenerator.close(); - - return reportPath; - } - - /** - * A CASE or UCO object. CASE objects can have properties and - * property bundles. - */ - private final class CASENode { - - private final String id; - private final String type; - - //Dynamic properties added to this CASENode. - private final Map properties; - private final List propertyBundle; - - public CASENode(String id, String type) { - this.id = id; - this.type = type; - properties = new LinkedHashMap<>(); - propertyBundle = new ArrayList<>(); - } - - @JsonProperty("@id") - public String getId() { - return id; - } - - @JsonProperty("@type") - public String getType() { - return type; - } - - @JsonAnyGetter - public Map getProperties() { - return properties; - } - - @JsonProperty("propertyBundle") - public List getPropertyBundle() { - return propertyBundle; - } - - public void addProperty(String key, Object val) { - properties.put(key, val); - } - - public void addBundle(CASEPropertyBundle bundle) { - propertyBundle.add(bundle); - } - } - - /** - * Contains CASE or UCO properties. - */ - private final class CASEPropertyBundle { - - private final Map properties; - - public CASEPropertyBundle(String type) { - properties = new LinkedHashMap<>(); - addProperty("@type", type); - } - - @JsonAnyGetter - public Map getProperties() { - return properties; - } - - public void addProperty(String key, Object val) { - properties.put(key, val); - } - } -} diff --git a/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java b/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java index d90b4a54c2..2da96ac34f 100755 --- a/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java +++ b/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java @@ -19,7 +19,15 @@ */ package org.sleuthkit.autopsy.report.modules.caseuco; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.stream.JsonWriter; + +import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -39,28 +47,36 @@ import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.report.GeneralReportModule; import org.sleuthkit.autopsy.report.GeneralReportSettings; import org.sleuthkit.autopsy.report.ReportProgressPanel; +import org.sleuthkit.caseuco.CaseUcoExporter; +import org.sleuthkit.caseuco.ContentNotExportableException; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; +import org.sleuthkit.datamodel.blackboardutils.attributes.BlackboardJsonAttrUtil; /** - * CaseUcoReportModule generates a report in CASE-UCO format. This module will + * CaseUcoReportModule generates a report in CASE-UCO format. This module will * write all files and data sources to the report. */ public final class CaseUcoReportModule implements GeneralReportModule { private static final Logger logger = Logger.getLogger(CaseUcoReportModule.class.getName()); private static final CaseUcoReportModule SINGLE_INSTANCE = new CaseUcoReportModule(); - - //Supported types of TSK_FS_FILES - private static final Set SUPPORTED_TYPES = new HashSet() {{ - add(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_UNDEF.getValue()); - add(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue()); - add(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_VIRT.getValue()); - }}; - private static final String REPORT_FILE_NAME = "CASE_UCO_output"; + //Supported types of TSK_FS_FILES + private static final Set SUPPORTED_TYPES = new HashSet() { + { + add(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_UNDEF.getValue()); + add(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue()); + add(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_VIRT.getValue()); + } + }; + + private static final String REPORT_FILE_NAME = "CASE_UCO_output"; private static final String EXTENSION = "json-ld"; // Hidden constructor for the report @@ -76,7 +92,7 @@ public final class CaseUcoReportModule implements GeneralReportModule { public String getName() { return NbBundle.getMessage(this.getClass(), "CaseUcoReportModule.getName.text"); } - + @Override public JPanel getConfigurationPanel() { return null; // No configuration panel @@ -84,7 +100,7 @@ public final class CaseUcoReportModule implements GeneralReportModule { @Override public String getRelativeFilePath() { - return REPORT_FILE_NAME + "." + EXTENSION; + return REPORT_FILE_NAME + "." + EXTENSION; } @Override @@ -100,7 +116,7 @@ public final class CaseUcoReportModule implements GeneralReportModule { public static String getReportFileName() { return REPORT_FILE_NAME; } - + @Override public boolean supportsDataSourceSelection() { return true; @@ -109,7 +125,7 @@ public final class CaseUcoReportModule implements GeneralReportModule { /** * Generates a CASE-UCO format report for all files in the Case. * - * @param settings Report settings. + * @param settings Report settings. * @param progressPanel panel to update the report's progress */ @NbBundle.Messages({ @@ -128,74 +144,106 @@ public final class CaseUcoReportModule implements GeneralReportModule { try { // Check if ingest has finished warnIngest(progressPanel); - + //Create report paths if they don't already exist. Path reportDirectory = Paths.get(settings.getReportDirectoryPath()); try { Files.createDirectories(reportDirectory); } catch (IOException ex) { logger.log(Level.WARNING, "Unable to create directory for CASE-UCO report.", ex); - progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, - Bundle.CaseUcoReportModule_unableToCreateDirectories()); + progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, + Bundle.CaseUcoReportModule_unableToCreateDirectories()); return; } - - CaseUcoReportGenerator generator = - new CaseUcoReportGenerator(reportDirectory, REPORT_FILE_NAME); - - //First write the Case to the report file. + Case caseObj = Case.getCurrentCaseThrows(); - generator.addCase(caseObj); - - List dataSources = caseObj.getDataSources().stream() - .filter((dataSource) -> { - if(settings.getSelectedDataSources() == null) { - // Assume all data sources if list is null. - return true; + + Path caseJsonReportFile = reportDirectory.resolve(REPORT_FILE_NAME + "." + EXTENSION); + + try (OutputStream stream = new FileOutputStream(caseJsonReportFile.toFile()); + JsonWriter reportWriter = new JsonWriter(new OutputStreamWriter(stream, "UTF-8"))) { + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + reportWriter.setIndent(" "); + reportWriter.beginObject(); + reportWriter.name("@graph"); + reportWriter.beginArray(); + + //First write the Case to the report file. + CaseUcoExporter exporter = new CaseUcoExporter(caseObj.getSleuthkitCase()); + for (JsonElement element : exporter.exportSleuthkitCase()) { + gson.toJson(element, reportWriter); + } + + // Prune the data sources so that we only report on what was selected. + List dataSources = caseObj.getSleuthkitCase().getDataSources().stream() + .filter((dataSource) -> { + if (settings.getSelectedDataSources() == null) { + // Assume all data sources if list is null. + return true; + } + return settings.getSelectedDataSources().contains(dataSource.getId()); + }) + .collect(Collectors.toList()); + + progressPanel.setIndeterminate(false); + progressPanel.setMaximumProgress(dataSources.size()); + progressPanel.start(); + + //Then search each data source for file content. + for (int i = 0; i < dataSources.size(); i++) { + DataSource dataSource = dataSources.get(i); + progressPanel.updateStatusLabel(String.format( + Bundle.CaseUcoReportModule_processingDataSource(), + dataSource.getName())); + // Add the data source export. + for (JsonElement element : exporter.exportDataSource(dataSource)) { + gson.toJson(element, reportWriter); + } + // Search all children of the data source. + performDepthFirstSearch(dataSource, gson, exporter, reportWriter); + progressPanel.setProgress(i + 1); + } + + // Write all standard artifacts to the report. + for (ARTIFACT_TYPE artType : caseObj.getSleuthkitCase().getBlackboardArtifactTypesInUse()) { + for (BlackboardArtifact artifact : caseObj.getSleuthkitCase().getBlackboardArtifacts(artType)) { + try { + for (JsonElement element : exporter.exportBlackboardArtifact(artifact)) { + gson.toJson(element, reportWriter); + } + } catch (ContentNotExportableException | BlackboardJsonAttrUtil.InvalidJsonException ex) { + logger.log(Level.WARNING, String.format("Unable to export blackboard artifact (id: %d) to CASE/UCO. " + + "The artifact type is either not supported or the artifact instance does not have any " + + "exportable attributes.", artifact.getId())); } - return settings.getSelectedDataSources().contains(dataSource.getId()); - }) - .collect(Collectors.toList()); - - progressPanel.setIndeterminate(false); - progressPanel.setMaximumProgress(dataSources.size()); - progressPanel.start(); - - //Then search each data source for file content. - for(int i = 0; i < dataSources.size(); i++) { - Content dataSource = dataSources.get(i); - progressPanel.updateStatusLabel(String.format( - Bundle.CaseUcoReportModule_processingDataSource(), - dataSource.getName())); - //Add the data source and then all children. - generator.addDataSource(dataSource, caseObj); - performDepthFirstSearch(dataSource, generator); - progressPanel.setProgress(i+1); + } + } + + reportWriter.endArray(); + reportWriter.endObject(); } - - //Complete the report. - Path reportPath = generator.generateReport(); - caseObj.addReport(reportPath.toString(), - Bundle.CaseUcoReportModule_srcModuleName(), + + caseObj.addReport(caseJsonReportFile.toString(), + Bundle.CaseUcoReportModule_srcModuleName(), REPORT_FILE_NAME); progressPanel.complete(ReportProgressPanel.ReportStatus.COMPLETE); } catch (IOException ex) { logger.log(Level.WARNING, "I/O error encountered while generating the report.", ex); - progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, + progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, Bundle.CaseUcoReportModule_ioError()); } catch (NoCurrentCaseException ex) { logger.log(Level.WARNING, "No case open.", ex); - progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, + progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, Bundle.CaseUcoReportModule_noCaseOpen()); } catch (TskCoreException ex) { logger.log(Level.WARNING, "TskCoreException encounted while generating the report.", ex); - progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, + progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, String.format(Bundle.CaseUcoReportModule_tskCoreException(), ex.toString())); } - + progressPanel.complete(ReportProgressPanel.ReportStatus.COMPLETE); } - + /** * Warn the user if ingest is still ongoing. */ @@ -204,28 +252,30 @@ public final class CaseUcoReportModule implements GeneralReportModule { progressPanel.updateStatusLabel(Bundle.CaseUcoReportModule_ingestWarning()); } } - + /** - * Perform DFS on the data sources tree, which will search it in entirety. - * This traversal is more memory efficient than BFS (Breadth first search). + * Perform DFS on the data sources tree, which will search it in entirety. */ - private void performDepthFirstSearch(Content dataSource, - CaseUcoReportGenerator generator) throws IOException, TskCoreException { - + private void performDepthFirstSearch(DataSource dataSource, + Gson gson, CaseUcoExporter exporter, JsonWriter reportWriter) throws IOException, TskCoreException { + Deque stack = new ArrayDeque<>(); stack.addAll(dataSource.getChildren()); //Depth First Search the data source tree. - while(!stack.isEmpty()) { + while (!stack.isEmpty()) { Content current = stack.pop(); - if(current instanceof AbstractFile) { - AbstractFile f = (AbstractFile) (current); - if(SUPPORTED_TYPES.contains(f.getMetaType().getValue())) { - generator.addFile(f, dataSource); + if (current instanceof AbstractFile) { + AbstractFile file = (AbstractFile) (current); + if (SUPPORTED_TYPES.contains(file.getMetaType().getValue())) { + + for (JsonElement element : exporter.exportAbstractFile(file)) { + gson.toJson(element, reportWriter); + } } } - for(Content child : current.getChildren()) { + for (Content child : current.getChildren()) { stack.push(child); } } diff --git a/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java b/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java index 8c88f29f53..b871d4bf84 100644 --- a/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java +++ b/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java @@ -20,12 +20,19 @@ package org.sleuthkit.autopsy.report.modules.portablecase; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.stream.JsonWriter; import org.sleuthkit.autopsy.report.ReportModule; import java.util.logging.Level; import java.io.BufferedReader; import java.io.File; +import java.io.FileOutputStream; import java.io.InputStreamReader; import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -49,7 +56,7 @@ import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.coreutils.FileTypeUtils.FileTypeCategory; import org.sleuthkit.autopsy.report.ReportProgressPanel; -import org.sleuthkit.autopsy.report.modules.caseuco.CaseUcoReportGenerator; +import org.sleuthkit.caseuco.CaseUcoExporter; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifactTag; @@ -76,6 +83,7 @@ import org.sleuthkit.datamodel.VolumeSystem; * Creates a portable case from tagged files */ public class PortableCaseReportModule implements ReportModule { + private static final Logger logger = Logger.getLogger(PortableCaseReportModule.class.getName()); private static final String FILE_FOLDER_NAME = "PortableCaseFiles"; // NON-NLS private static final String UNKNOWN_FILE_TYPE_FOLDER = "Other"; // NON-NLS @@ -83,35 +91,35 @@ public class PortableCaseReportModule implements ReportModule { private static final String CASE_UCO_FILE_NAME = "portable_CASE_UCO_output"; private static final String CASE_UCO_TMP_DIR = "case_uco_tmp"; private PortableCaseReportModuleSettings settings; - + // These are the types for the exported file subfolders private static final List FILE_TYPE_CATEGORIES = Arrays.asList(FileTypeCategory.AUDIO, FileTypeCategory.DOCUMENTS, FileTypeCategory.EXECUTABLE, FileTypeCategory.IMAGE, FileTypeCategory.VIDEO); - + private Case currentCase = null; private SleuthkitCase portableSkCase = null; private String caseName = ""; private File caseFolder = null; private File copiedFilesFolder = null; - + // Maps old object ID from current case to new object in portable case private final Map oldIdToNewContent = new HashMap<>(); - + // Maps new object ID to the new object private final Map newIdToContent = new HashMap<>(); - + // Maps old TagName to new TagName private final Map oldTagNameToNewTagName = new HashMap<>(); // Map of old artifact type ID to new artifact type ID. There will only be changes if custom artifact types are present. private final Map oldArtTypeIdToNewArtTypeId = new HashMap<>(); - + // Map of old attribute type ID to new attribute type ID. There will only be changes if custom attr types are present. private final Map oldAttrTypeIdToNewAttrType = new HashMap<>(); - + // Map of old artifact ID to new artifact private final Map oldArtifactIdToNewArtifact = new HashMap<>(); - + public PortableCaseReportModule() { } @@ -141,11 +149,11 @@ public class PortableCaseReportModule implements ReportModule { } return caseName; } - + /** * Convenience method for handling cancellation - * - * @param progressPanel The report progress panel + * + * @param progressPanel The report progress panel */ private void handleCancellation(ReportProgressPanel progressPanel) { logger.log(Level.INFO, "Portable case creation canceled by user"); // NON-NLS @@ -153,16 +161,16 @@ public class PortableCaseReportModule implements ReportModule { progressPanel.complete(ReportProgressPanel.ReportStatus.CANCELED); cleanup(); } - + /** - * Convenience method to avoid code duplication. - * Assumes that if an exception is supplied then the error is SEVERE. Otherwise - * it is logged as a WARNING. - * - * @param logWarning Warning to write to the log - * @param dialogWarning Warning to write to a pop-up window - * @param ex The exception (can be null) - * @param progressPanel The report progress panel + * Convenience method to avoid code duplication. Assumes that if an + * exception is supplied then the error is SEVERE. Otherwise it is logged as + * a WARNING. + * + * @param logWarning Warning to write to the log + * @param dialogWarning Warning to write to a pop-up window + * @param ex The exception (can be null) + * @param progressPanel The report progress panel */ private void handleError(String logWarning, String dialogWarning, Exception ex, ReportProgressPanel progressPanel) { if (ex == null) { @@ -208,24 +216,24 @@ public class PortableCaseReportModule implements ReportModule { progressPanel.setIndeterminate(true); progressPanel.start(); progressPanel.updateStatusLabel(Bundle.PortableCaseReportModule_generateReport_verifying()); - + // Clear out any old values cleanup(); - + // Validate the input parameters File outputDir = new File(reportPath); - if (! outputDir.exists()) { + if (!outputDir.exists()) { handleError("Output folder " + outputDir.toString() + " does not exist", Bundle.PortableCaseReportModule_generateReport_outputDirDoesNotExist(outputDir.toString()), null, progressPanel); // NON-NLS return; } - - if (! outputDir.isDirectory()) { + + if (!outputDir.isDirectory()) { handleError("Output folder " + outputDir.toString() + " is not a folder", Bundle.PortableCaseReportModule_generateReport_outputDirIsNotDir(outputDir.toString()), null, progressPanel); // NON-NLS return; } - + // Save the current case object try { currentCase = Case.getCurrentCaseThrows(); @@ -234,41 +242,41 @@ public class PortableCaseReportModule implements ReportModule { handleError("Current case has been closed", Bundle.PortableCaseReportModule_generateReport_caseClosed(), null, progressPanel); // NON-NLS return; - } - + } + // Check that there will be something to copy List tagNames; if (options.areAllTagsSelected()) { try { tagNames = Case.getCurrentCaseThrows().getServices().getTagsManager().getTagNamesInUse(); } catch (NoCurrentCaseException | TskCoreException ex) { - handleError("Unable to get all tags", - Bundle.PortableCaseReportModule_generateReport_errorReadingTags(), ex, progressPanel); // NON-NLS + handleError("Unable to get all tags", + Bundle.PortableCaseReportModule_generateReport_errorReadingTags(), ex, progressPanel); // NON-NLS return; } } else { tagNames = options.getSelectedTagNames(); } - + List setNames; if (options.areAllSetsSelected()) { try { setNames = getAllInterestingItemsSets(); } catch (NoCurrentCaseException | TskCoreException ex) { - handleError("Unable to get all interesting items sets", - Bundle.PortableCaseReportModule_generateReport_errorReadingSets(), ex, progressPanel); // NON-NLS + handleError("Unable to get all interesting items sets", + Bundle.PortableCaseReportModule_generateReport_errorReadingSets(), ex, progressPanel); // NON-NLS return; } } else { setNames = options.getSelectedSetNames(); } - - if (tagNames.isEmpty() && setNames.isEmpty()) { - handleError("No content to copy", + + if (tagNames.isEmpty() && setNames.isEmpty()) { + handleError("No content to copy", Bundle.PortableCaseReportModule_generateReport_noContentToCopy(), null, progressPanel); // NON-NLS return; } - + // Create the case. // portableSkCase and caseFolder will be set here. progressPanel.updateStatusLabel(Bundle.PortableCaseReportModule_generateReport_creatingCase()); @@ -277,13 +285,13 @@ public class PortableCaseReportModule implements ReportModule { // The error has already been handled return; } - + // Check for cancellation if (progressPanel.getStatus() == ReportProgressPanel.ReportStatus.CANCELED) { handleCancellation(progressPanel); return; } - + // Set up the table for the image tags try { initializeImageTags(progressPanel); @@ -291,11 +299,11 @@ public class PortableCaseReportModule implements ReportModule { handleError("Error creating image tag table", Bundle.PortableCaseReportModule_generateReport_errorCreatingImageTagTable(), ex, progressPanel); // NON-NLS return; } - + // Copy the selected tags progressPanel.updateStatusLabel(Bundle.PortableCaseReportModule_generateReport_copyingTags()); try { - for(TagName tagName:tagNames) { + for (TagName tagName : tagNames) { TagName newTagName = portableSkCase.addOrUpdateTagName(tagName.getDisplayName(), tagName.getDescription(), tagName.getColor(), tagName.getKnownStatus()); oldTagNameToNewTagName.put(tagName, newTagName); } @@ -303,12 +311,12 @@ public class PortableCaseReportModule implements ReportModule { handleError("Error copying tags", Bundle.PortableCaseReportModule_generateReport_errorCopyingTags(), ex, progressPanel); // NON-NLS return; } - + // Set up tracking to support any custom artifact or attribute types - for (BlackboardArtifact.ARTIFACT_TYPE type:BlackboardArtifact.ARTIFACT_TYPE.values()) { + for (BlackboardArtifact.ARTIFACT_TYPE type : BlackboardArtifact.ARTIFACT_TYPE.values()) { oldArtTypeIdToNewArtTypeId.put(type.getTypeID(), type.getTypeID()); } - for (BlackboardAttribute.ATTRIBUTE_TYPE type:BlackboardAttribute.ATTRIBUTE_TYPE.values()) { + for (BlackboardAttribute.ATTRIBUTE_TYPE type : BlackboardAttribute.ATTRIBUTE_TYPE.values()) { try { oldAttrTypeIdToNewAttrType.put(type.getTypeID(), portableSkCase.getAttributeType(type.getLabel())); } catch (TskCoreException ex) { @@ -316,11 +324,11 @@ public class PortableCaseReportModule implements ReportModule { Bundle.PortableCaseReportModule_generateReport_errorLookingUpAttrType(type.getLabel()), ex, progressPanel); // NON-NLS } - } - + } + // Copy the tagged files try { - for(TagName tagName:tagNames) { + for (TagName tagName : tagNames) { // Check for cancellation if (progressPanel.getStatus() == ReportProgressPanel.ReportStatus.CANCELED) { handleCancellation(progressPanel); @@ -328,7 +336,7 @@ public class PortableCaseReportModule implements ReportModule { } progressPanel.updateStatusLabel(Bundle.PortableCaseReportModule_generateReport_copyingFiles(tagName.getDisplayName())); addFilesToPortableCase(tagName, progressPanel); - + // Check for cancellation if (progressPanel.getStatus() == ReportProgressPanel.ReportStatus.CANCELED) { handleCancellation(progressPanel); @@ -338,11 +346,11 @@ public class PortableCaseReportModule implements ReportModule { } catch (TskCoreException ex) { handleError("Error copying tagged files", Bundle.PortableCaseReportModule_generateReport_errorCopyingFiles(), ex, progressPanel); // NON-NLS return; - } - + } + // Copy the tagged artifacts and associated files try { - for(TagName tagName:tagNames) { + for (TagName tagName : tagNames) { // Check for cancellation if (progressPanel.getStatus() == ReportProgressPanel.ReportStatus.CANCELED) { handleCancellation(progressPanel); @@ -350,7 +358,7 @@ public class PortableCaseReportModule implements ReportModule { } progressPanel.updateStatusLabel(Bundle.PortableCaseReportModule_generateReport_copyingArtifacts(tagName.getDisplayName())); addArtifactsToPortableCase(tagName, progressPanel); - + // Check for cancellation if (progressPanel.getStatus() == ReportProgressPanel.ReportStatus.CANCELED) { handleCancellation(progressPanel); @@ -361,18 +369,18 @@ public class PortableCaseReportModule implements ReportModule { handleError("Error copying tagged artifacts", Bundle.PortableCaseReportModule_generateReport_errorCopyingArtifacts(), ex, progressPanel); // NON-NLS return; } - + // Copy interesting files and results - if (! setNames.isEmpty()) { + if (!setNames.isEmpty()) { try { List interestingFiles = currentCase.getSleuthkitCase().getBlackboardArtifacts(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT); - for (BlackboardArtifact art:interestingFiles) { + for (BlackboardArtifact art : interestingFiles) { // Check for cancellation if (progressPanel.getStatus() == ReportProgressPanel.ReportStatus.CANCELED) { handleCancellation(progressPanel); return; } - + BlackboardAttribute setAttr = art.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME)); if (setNames.contains(setAttr.getValueString())) { copyContentToPortableCase(art, progressPanel); @@ -385,7 +393,7 @@ public class PortableCaseReportModule implements ReportModule { try { List interestingResults = currentCase.getSleuthkitCase().getBlackboardArtifacts(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT); - for (BlackboardArtifact art:interestingResults) { + for (BlackboardArtifact art : interestingResults) { // Check for cancellation if (progressPanel.getStatus() == ReportProgressPanel.ReportStatus.CANCELED) { handleCancellation(progressPanel); @@ -400,49 +408,49 @@ public class PortableCaseReportModule implements ReportModule { handleError("Error copying interesting results", Bundle.PortableCaseReportModule_generateReport_errorCopyingInterestingResults(), ex, progressPanel); // NON-NLS return; } - } - + } + // Check for cancellation if (progressPanel.getStatus() == ReportProgressPanel.ReportStatus.CANCELED) { handleCancellation(progressPanel); return; } - + //Attempt to generate and included the CASE-UCO report. generateCaseUcoReport(tagNames, setNames, progressPanel); // Compress the case (if desired) if (options.shouldCompress()) { progressPanel.updateStatusLabel(Bundle.PortableCaseReportModule_generateReport_compressingCase()); - + boolean success = compressCase(progressPanel); - + // Check for cancellation if (progressPanel.getStatus() == ReportProgressPanel.ReportStatus.CANCELED) { handleCancellation(progressPanel); return; } - - if (! success) { + + if (!success) { // Errors have been handled already return; } } - + // Close the case connections and clear out the maps cleanup(); - + progressPanel.complete(ReportProgressPanel.ReportStatus.COMPLETE); - + } - + /** * Generates a CASE-UCO report for all files that have a specified TagName * or TSK_INTERESTING artifacts that are flagged by the specified SET_NAMEs. - * + * * Only one copy of the file will be saved in the report if it is the source * of more than one of the above. - * + * * @param tagNames TagNames to included in the report. * @param setNames SET_NAMEs to include in the report. * @param progressPanel ProgressPanel to relay progress messages. @@ -456,17 +464,24 @@ public class PortableCaseReportModule implements ReportModule { private void generateCaseUcoReport(List tagNames, List setNames, ReportProgressPanel progressPanel) { //Create the 'Reports' directory to include a CASE-UCO report. Path reportsDirectory = Paths.get(caseFolder.toString(), "Reports"); - if(!reportsDirectory.toFile().mkdir()) { + if (!reportsDirectory.toFile().mkdir()) { logger.log(Level.SEVERE, "Could not make the report folder... skipping " + "CASE-UCO report generation for the portable case"); return; } - try { - //Try to generate case uco output. - progressPanel.updateStatusLabel(Bundle.PortableCaseReportModule_generateCaseUcoReport_startCaseUcoReportGeneration()); - CaseUcoReportGenerator reportGenerator = new CaseUcoReportGenerator(reportsDirectory, CASE_UCO_FILE_NAME); - //Acquire references for file discovery + Path reportFile = reportsDirectory.resolve(CASE_UCO_FILE_NAME); + + progressPanel.updateStatusLabel(Bundle.PortableCaseReportModule_generateCaseUcoReport_startCaseUcoReportGeneration()); + try (OutputStream stream = new FileOutputStream(reportFile.toFile()); + JsonWriter reportWriter = new JsonWriter(new OutputStreamWriter(stream, "UTF-8"))) { + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + reportWriter.setIndent(" "); + reportWriter.beginObject(); + reportWriter.name("@graph"); + reportWriter.beginArray(); + + //Acquire references to querying files. String caseTempDirectory = currentCase.getTempDirectory(); SleuthkitCase skCase = currentCase.getSleuthkitCase(); TagsManager tagsManager = currentCase.getServices().getTagsManager(); @@ -477,43 +492,45 @@ public class PortableCaseReportModule implements ReportModule { FileUtils.deleteDirectory(tmpDir.toFile()); Files.createDirectory(tmpDir); - reportGenerator.addCase(currentCase); - + //First export the Sleuthkit Case instance. + CaseUcoExporter exporter = new CaseUcoExporter(currentCase.getSleuthkitCase()); + for (JsonElement element : exporter.exportSleuthkitCase()) { + gson.toJson(element, reportWriter); + } + //Load all interesting BlackboardArtifacts that belong to the selected SET_NAMEs //binned by data source id. Multimap artifactsWithSetName = getInterestingArtifactsBySetName(skCase, setNames); - + //Search each data source looking for content tags and interesting //items that match the selected tag names and set names. - for (Content dataSource : currentCase.getDataSources()) { - /** - * It is currently believed that DataSources in a CASE-UCO report - * should precede all file entities. Therefore, before - * writing a file, add the data source if it - * has yet to be included. - */ + for (DataSource dataSource : currentCase.getSleuthkitCase().getDataSources()) { + // Helper flag to ensure each data source is only written once in + // a report. boolean dataSourceHasBeenIncluded = false; + //Search content tags and artifact tags that match for (TagName tagName : tagNames) { for (ContentTag ct : tagsManager.getContentTagsByTagName(tagName, dataSource.getId())) { - dataSourceHasBeenIncluded |= addUniqueFile(ct.getContent(), - dataSource, tmpDir, reportGenerator, dataSourceHasBeenIncluded); + dataSourceHasBeenIncluded |= addUniqueFile(ct.getContent(), + dataSource, tmpDir, gson, exporter, reportWriter, dataSourceHasBeenIncluded); } for (BlackboardArtifactTag bat : tagsManager.getBlackboardArtifactTagsByTagName(tagName, dataSource.getId())) { - dataSourceHasBeenIncluded |= addUniqueFile(bat.getContent(), - dataSource, tmpDir, reportGenerator, dataSourceHasBeenIncluded); + dataSourceHasBeenIncluded |= addUniqueFile(bat.getContent(), + dataSource, tmpDir, gson, exporter, reportWriter, dataSourceHasBeenIncluded); } } //Search artifacts that this data source contains - for(BlackboardArtifact bArt : artifactsWithSetName.get(dataSource.getId())) { + for (BlackboardArtifact bArt : artifactsWithSetName.get(dataSource.getId())) { Content sourceContent = bArt.getParent(); - dataSourceHasBeenIncluded |= addUniqueFile(sourceContent, dataSource, - tmpDir, reportGenerator, dataSourceHasBeenIncluded); + dataSourceHasBeenIncluded |= addUniqueFile(sourceContent, dataSource, + tmpDir, gson, exporter, reportWriter, dataSourceHasBeenIncluded); } } - - //Create the report. - reportGenerator.generateReport(); + + // Finish the report. + reportWriter.endArray(); + reportWriter.endObject(); progressPanel.updateStatusLabel(Bundle.PortableCaseReportModule_generateCaseUcoReport_successCaseUcoReportGeneration()); } catch (IOException | TskCoreException ex) { progressPanel.updateStatusLabel(Bundle.PortableCaseReportModule_generateCaseUcoReport_errorGeneratingCaseUcoReport()); @@ -522,7 +539,7 @@ public class PortableCaseReportModule implements ReportModule { + "completed without a CASE-UCO report.", ex); } } - + /** * Load all interesting BlackboardArtifacts that belong to the selected * SET_NAME. This operation would be duplicated for every data source, since @@ -530,15 +547,15 @@ public class PortableCaseReportModule implements ReportModule { */ private Multimap getInterestingArtifactsBySetName(SleuthkitCase skCase, List setNames) throws TskCoreException { Multimap artifactsWithSetName = ArrayListMultimap.create(); - if(!setNames.isEmpty()) { + if (!setNames.isEmpty()) { List allArtifacts = skCase.getBlackboardArtifacts( BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT); allArtifacts.addAll(skCase.getBlackboardArtifacts( BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT)); - for(BlackboardArtifact bArt : allArtifacts) { + for (BlackboardArtifact bArt : allArtifacts) { BlackboardAttribute setAttr = bArt.getAttribute( - new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME)); + new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME)); if (setNames.contains(setAttr.getValueString())) { artifactsWithSetName.put(bArt.getDataSource().getId(), bArt); } @@ -546,7 +563,7 @@ public class PortableCaseReportModule implements ReportModule { } return artifactsWithSetName; } - + /** * Adds the content if and only if it has not already been seen. * @@ -555,32 +572,36 @@ public class PortableCaseReportModule implements ReportModule { * @param tmpDir Path to the tmpDir to enforce uniqueness * @param reportGenerator Report generator instance to add the content to * @param dataSourceHasBeenIncluded Flag determining if the data source - * should be written before the file. False will cause the data source to be written. + * should be written to the report (false indicates that it should be written). * @throws IOException If an I/O error occurs. * @throws TskCoreException If an internal database error occurs. * - * return True if the data source was written during this operation. + * return True if the file was written during this operation. */ - private boolean addUniqueFile(Content content, Content dataSource, - Path tmpDir, CaseUcoReportGenerator reportGenerator, + private boolean addUniqueFile(Content content, DataSource dataSource, + Path tmpDir, Gson gson, CaseUcoExporter exporter, JsonWriter reportWriter, boolean dataSourceHasBeenIncluded) throws IOException, TskCoreException { if (content instanceof AbstractFile && !(content instanceof DataSource)) { AbstractFile absFile = (AbstractFile) content; Path filePath = tmpDir.resolve(Long.toString(absFile.getId())); if (!absFile.isDir() && !Files.exists(filePath)) { - if(!dataSourceHasBeenIncluded) { - reportGenerator.addDataSource(dataSource, currentCase); + if (!dataSourceHasBeenIncluded) { + for (JsonElement element : exporter.exportDataSource(dataSource)) { + gson.toJson(element, reportWriter); + } } String subFolder = getExportSubfolder(absFile); String fileName = absFile.getId() + "-" + FileUtil.escapeFileName(absFile.getName()); - reportGenerator.addFile(absFile, dataSource, Paths.get(FILE_FOLDER_NAME, subFolder, fileName)); + for (JsonElement element : exporter.exportAbstractFile(absFile, Paths.get(FILE_FOLDER_NAME, subFolder, fileName).toString())) { + gson.toJson(element, reportWriter); + } Files.createFile(filePath); return true; } } return false; } - + private List getAllInterestingItemsSets() throws NoCurrentCaseException, TskCoreException { // Get the set names in use for the current case. @@ -603,14 +624,13 @@ public class PortableCaseReportModule implements ReportModule { setNames.addAll(setCounts.keySet()); return setNames; } - /** - * Create the case directory and case database. - * portableSkCase will be set if this completes without error. - * - * @param outputDir The parent for the case folder - * @param progressPanel + * Create the case directory and case database. portableSkCase will be set + * if this completes without error. + * + * @param outputDir The parent for the case folder + * @param progressPanel */ @NbBundle.Messages({ "# {0} - case folder", @@ -618,8 +638,7 @@ public class PortableCaseReportModule implements ReportModule { "PortableCaseReportModule.createCase.errorCreatingCase=Error creating case", "# {0} - folder", "PortableCaseReportModule.createCase.errorCreatingFolder=Error creating folder {0}", - "PortableCaseReportModule.createCase.errorStoringMaxIds=Error storing maximum database IDs", - }) + "PortableCaseReportModule.createCase.errorStoringMaxIds=Error storing maximum database IDs",}) private void createCase(File outputDir, ReportProgressPanel progressPanel) { // Create the case folder @@ -627,66 +646,66 @@ public class PortableCaseReportModule implements ReportModule { if (caseFolder.exists()) { handleError("Case folder " + caseFolder.toString() + " already exists", - Bundle.PortableCaseReportModule_createCase_caseDirExists(caseFolder.toString()), null, progressPanel); // NON-NLS + Bundle.PortableCaseReportModule_createCase_caseDirExists(caseFolder.toString()), null, progressPanel); // NON-NLS return; } - + // Create the case try { portableSkCase = currentCase.createPortableCase(caseName, caseFolder); } catch (TskCoreException ex) { handleError("Error creating case " + caseName + " in folder " + caseFolder.toString(), - Bundle.PortableCaseReportModule_createCase_errorCreatingCase(), ex, progressPanel); // NON-NLS + Bundle.PortableCaseReportModule_createCase_errorCreatingCase(), ex, progressPanel); // NON-NLS return; } - + // Store the highest IDs try { saveHighestIds(); } catch (TskCoreException ex) { handleError("Error storing maximum database IDs", - Bundle.PortableCaseReportModule_createCase_errorStoringMaxIds(), ex, progressPanel); // NON-NLS + Bundle.PortableCaseReportModule_createCase_errorStoringMaxIds(), ex, progressPanel); // NON-NLS return; } - + // Create the base folder for the copied files copiedFilesFolder = Paths.get(caseFolder.toString(), FILE_FOLDER_NAME).toFile(); - if (! copiedFilesFolder.mkdir()) { + if (!copiedFilesFolder.mkdir()) { handleError("Error creating folder " + copiedFilesFolder.toString(), Bundle.PortableCaseReportModule_createCase_errorCreatingFolder(copiedFilesFolder.toString()), null, progressPanel); // NON-NLS return; } - + // Create subfolders for the copied files - for (FileTypeCategory cat:FILE_TYPE_CATEGORIES) { + for (FileTypeCategory cat : FILE_TYPE_CATEGORIES) { File subFolder = Paths.get(copiedFilesFolder.toString(), cat.getDisplayName()).toFile(); - if (! subFolder.mkdir()) { + if (!subFolder.mkdir()) { handleError("Error creating folder " + subFolder.toString(), - Bundle.PortableCaseReportModule_createCase_errorCreatingFolder(subFolder.toString()), null, progressPanel); // NON-NLS + Bundle.PortableCaseReportModule_createCase_errorCreatingFolder(subFolder.toString()), null, progressPanel); // NON-NLS return; } } File unknownTypeFolder = Paths.get(copiedFilesFolder.toString(), UNKNOWN_FILE_TYPE_FOLDER).toFile(); - if (! unknownTypeFolder.mkdir()) { + if (!unknownTypeFolder.mkdir()) { handleError("Error creating folder " + unknownTypeFolder.toString(), - Bundle.PortableCaseReportModule_createCase_errorCreatingFolder(unknownTypeFolder.toString()), null, progressPanel); // NON-NLS + Bundle.PortableCaseReportModule_createCase_errorCreatingFolder(unknownTypeFolder.toString()), null, progressPanel); // NON-NLS return; } - + } - + /** * Save the current highest IDs to the portable case. - * - * @throws TskCoreException + * + * @throws TskCoreException */ private void saveHighestIds() throws TskCoreException { - + CaseDbAccessManager currentCaseDbManager = currentCase.getSleuthkitCase().getCaseDbAccessManager(); - + String tableSchema = "( table_name TEXT PRIMARY KEY, " - + " max_id TEXT)"; // NON-NLS - + + " max_id TEXT)"; // NON-NLS + portableSkCase.getCaseDbAccessManager().createTable(MAX_ID_TABLE_NAME, tableSchema); currentCaseDbManager.select("max(obj_id) as max_id from tsk_objects", new StoreMaxIdCallback("tsk_objects")); // NON-NLS @@ -694,51 +713,51 @@ public class PortableCaseReportModule implements ReportModule { currentCaseDbManager.select("max(tag_id) as max_id from blackboard_artifact_tags", new StoreMaxIdCallback("blackboard_artifact_tags")); // NON-NLS currentCaseDbManager.select("max(examiner_id) as max_id from tsk_examiners", new StoreMaxIdCallback("tsk_examiners")); // NON-NLS } - + /** * Set up the image tag table in the portable case - * - * @param progressPanel - * - * @throws TskCoreException + * + * @param progressPanel + * + * @throws TskCoreException */ private void initializeImageTags(ReportProgressPanel progressPanel) throws TskCoreException { - + // Create the image tags table in the portable case CaseDbAccessManager portableDbAccessManager = portableSkCase.getCaseDbAccessManager(); - if (! portableDbAccessManager.tableExists(ContentViewerTagManager.TABLE_NAME)) { + if (!portableDbAccessManager.tableExists(ContentViewerTagManager.TABLE_NAME)) { portableDbAccessManager.createTable(ContentViewerTagManager.TABLE_NAME, ContentViewerTagManager.TABLE_SCHEMA_SQLITE); } } - + /** * Add all files with a given tag to the portable case. - * - * @param oldTagName The TagName object from the current case + * + * @param oldTagName The TagName object from the current case * @param progressPanel The progress panel - * - * @throws TskCoreException + * + * @throws TskCoreException */ private void addFilesToPortableCase(TagName oldTagName, ReportProgressPanel progressPanel) throws TskCoreException { - + // Get all the tags in the current case List tags = currentCase.getServices().getTagsManager().getContentTagsByTagName(oldTagName); - + // Copy the files into the portable case and tag for (ContentTag tag : tags) { - + // Check for cancellation if (progressPanel.getStatus() == ReportProgressPanel.ReportStatus.CANCELED) { return; } - + Content content = tag.getContent(); if (content instanceof AbstractFile) { - + long newFileId = copyContentToPortableCase(content, progressPanel); - + // Tag the file - if (! oldTagNameToNewTagName.containsKey(tag.getName())) { + if (!oldTagNameToNewTagName.containsKey(tag.getName())) { throw new TskCoreException("TagName map is missing entry for ID " + tag.getName().getId() + " with display name " + tag.getName().getDisplayName()); // NON-NLS } ContentTagChange newContentTag = portableSkCase.getTaggingManager().addContentTag(newIdToContent.get(newFileId), oldTagNameToNewTagName.get(tag.getName()), tag.getComment(), tag.getBeginByteOffset(), tag.getEndByteOffset()); @@ -746,21 +765,22 @@ public class PortableCaseReportModule implements ReportModule { // Get the image tag data associated with this tag (empty string if there is none) // and save it if present String appData = getImageTagDataForContentTag(tag); - if (! appData.isEmpty()) { + if (!appData.isEmpty()) { addImageTagToPortableCase(newContentTag.getAddedTag(), appData); } } - } - } - + } + } + /** * Gets the image tag data for a given content tag - * + * * @param tag The ContentTag in the current case - * - * @return The app_data string for this content tag or an empty string if there was none - * - * @throws TskCoreException + * + * @return The app_data string for this content tag or an empty string if + * there was none + * + * @throws TskCoreException */ private String getImageTagDataForContentTag(ContentTag tag) throws TskCoreException { @@ -769,7 +789,7 @@ public class PortableCaseReportModule implements ReportModule { currentCase.getSleuthkitCase().getCaseDbAccessManager().select(query, callback); return callback.getAppData(); } - + /** * CaseDbAccessManager callback to get the app_data string for the image tag */ @@ -777,7 +797,7 @@ public class PortableCaseReportModule implements ReportModule { private static final Logger logger = Logger.getLogger(PortableCaseReportModule.class.getName()); private String appData = ""; - + @Override public void process(ResultSet rs) { try { @@ -791,106 +811,107 @@ public class PortableCaseReportModule implements ReportModule { } catch (SQLException ex) { logger.log(Level.WARNING, "Failed to get next result for app_data", ex); // NON-NLS } - } - + } + /** * Get the app_data string - * + * * @return the app_data string */ String getAppData() { return appData; } } - + /** * Add an image tag to the portable case. - * + * * @param newContentTag The content tag in the portable case - * @param appData The string to copy into app_data - * - * @throws TskCoreException + * @param appData The string to copy into app_data + * + * @throws TskCoreException */ private void addImageTagToPortableCase(ContentTag newContentTag, String appData) throws TskCoreException { String insert = "(content_tag_id, app_data) VALUES (" + newContentTag.getId() + ", '" + appData + "')"; portableSkCase.getCaseDbAccessManager().insert(ContentViewerTagManager.TABLE_NAME, insert); } - - + /** * Add all artifacts with a given tag to the portable case. - * - * @param oldTagName The TagName object from the current case + * + * @param oldTagName The TagName object from the current case * @param progressPanel The progress panel - * - * @throws TskCoreException + * + * @throws TskCoreException */ private void addArtifactsToPortableCase(TagName oldTagName, ReportProgressPanel progressPanel) throws TskCoreException { - + List tags = currentCase.getServices().getTagsManager().getBlackboardArtifactTagsByTagName(oldTagName); - + // Copy the artifacts into the portable case along with their content and tag for (BlackboardArtifactTag tag : tags) { - + // Check for cancellation if (progressPanel.getStatus() == ReportProgressPanel.ReportStatus.CANCELED) { return; } - + // Copy the source content Content content = tag.getContent(); long newContentId = copyContentToPortableCase(content, progressPanel); - + // Copy the artifact BlackboardArtifact newArtifact = copyArtifact(newContentId, tag.getArtifact()); - + // Tag the artfiact - if (! oldTagNameToNewTagName.containsKey(tag.getName())) { + if (!oldTagNameToNewTagName.containsKey(tag.getName())) { throw new TskCoreException("TagName map is missing entry for ID " + tag.getName().getId() + " with display name " + tag.getName().getDisplayName()); // NON-NLS } portableSkCase.getTaggingManager().addArtifactTag(newArtifact, oldTagNameToNewTagName.get(tag.getName()), tag.getComment()); - } - } - + } + } + /** - * Copy an artifact into the new case. Will also copy any associated artifacts - * - * @param newContentId The content ID (in the portable case) of the source content + * Copy an artifact into the new case. Will also copy any associated + * artifacts + * + * @param newContentId The content ID (in the portable case) of the source + * content * @param artifactToCopy The artifact to copy - * + * * @return The new artifact in the portable case - * - * @throws TskCoreException + * + * @throws TskCoreException */ private BlackboardArtifact copyArtifact(long newContentId, BlackboardArtifact artifactToCopy) throws TskCoreException { - + if (oldArtifactIdToNewArtifact.containsKey(artifactToCopy.getArtifactID())) { return oldArtifactIdToNewArtifact.get(artifactToCopy.getArtifactID()); } - + // First create the associated artifact (if present) BlackboardAttribute oldAssociatedAttribute = artifactToCopy.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT)); List newAttrs = new ArrayList<>(); if (oldAssociatedAttribute != null) { BlackboardArtifact oldAssociatedArtifact = currentCase.getSleuthkitCase().getBlackboardArtifact(oldAssociatedAttribute.getValueLong()); BlackboardArtifact newAssociatedArtifact = copyArtifact(newContentId, oldAssociatedArtifact); - newAttrs.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, - String.join(",", oldAssociatedAttribute.getSources()), newAssociatedArtifact.getArtifactID())); + newAttrs.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, + String.join(",", oldAssociatedAttribute.getSources()), newAssociatedArtifact.getArtifactID())); } - + // Create the new artifact int newArtifactTypeId = getNewArtifactTypeId(artifactToCopy); BlackboardArtifact newArtifact = portableSkCase.newBlackboardArtifact(newArtifactTypeId, newContentId); List oldAttrs = artifactToCopy.getAttributes(); - + // Copy over each attribute, making sure the type is in the new case. - for (BlackboardAttribute oldAttr:oldAttrs) { - + for (BlackboardAttribute oldAttr : oldAttrs) { + // The associated artifact has already been handled if (oldAttr.getAttributeType().getTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT.getTypeID()) { continue; } - + BlackboardAttribute.Type newAttributeType = getNewAttributeType(oldAttr); switch (oldAttr.getValueType()) { case BYTE: @@ -905,7 +926,7 @@ public class PortableCaseReportModule implements ReportModule { newAttrs.add(new BlackboardAttribute(newAttributeType, String.join(",", oldAttr.getSources()), oldAttr.getValueInt())); break; - case DATETIME: + case DATETIME: case LONG: newAttrs.add(new BlackboardAttribute(newAttributeType, String.join(",", oldAttr.getSources()), oldAttr.getValueLong())); @@ -919,26 +940,27 @@ public class PortableCaseReportModule implements ReportModule { throw new TskCoreException("Unexpected attribute value type found: " + oldAttr.getValueType().getLabel()); // NON-NLS } } - + newArtifact.addAttributes(newAttrs); - + oldArtifactIdToNewArtifact.put(artifactToCopy.getArtifactID(), newArtifact); return newArtifact; } - + /** - * Get the artifact type ID in the portable case and create new artifact type if needed. - * For built-in artifacts this will be the same as the original. - * + * Get the artifact type ID in the portable case and create new artifact + * type if needed. For built-in artifacts this will be the same as the + * original. + * * @param oldArtifact The artifact in the current case - * + * * @return The corresponding artifact type ID in the portable case */ private int getNewArtifactTypeId(BlackboardArtifact oldArtifact) throws TskCoreException { if (oldArtTypeIdToNewArtTypeId.containsKey(oldArtifact.getArtifactTypeID())) { return oldArtTypeIdToNewArtTypeId.get(oldArtifact.getArtifactTypeID()); } - + BlackboardArtifact.Type oldCustomType = currentCase.getSleuthkitCase().getArtifactType(oldArtifact.getArtifactTypeName()); try { BlackboardArtifact.Type newCustomType = portableSkCase.addBlackboardArtifactType(oldCustomType.getTypeName(), oldCustomType.getDisplayName()); @@ -948,13 +970,14 @@ public class PortableCaseReportModule implements ReportModule { throw new TskCoreException("Error creating new artifact type " + oldCustomType.getTypeName(), ex); // NON-NLS } } - + /** - * Get the attribute type ID in the portable case and create new attribute type if needed. - * For built-in attributes this will be the same as the original. - * + * Get the attribute type ID in the portable case and create new attribute + * type if needed. For built-in attributes this will be the same as the + * original. + * * @param oldAttribute The attribute in the current case - * + * * @return The corresponding attribute type in the portable case */ private BlackboardAttribute.Type getNewAttributeType(BlackboardAttribute oldAttribute) throws TskCoreException { @@ -962,9 +985,9 @@ public class PortableCaseReportModule implements ReportModule { if (oldAttrTypeIdToNewAttrType.containsKey(oldAttrType.getTypeID())) { return oldAttrTypeIdToNewAttrType.get(oldAttrType.getTypeID()); } - + try { - BlackboardAttribute.Type newCustomType = portableSkCase.addArtifactAttributeType(oldAttrType.getTypeName(), + BlackboardAttribute.Type newCustomType = portableSkCase.addArtifactAttributeType(oldAttrType.getTypeName(), oldAttrType.getValueType(), oldAttrType.getDisplayName()); oldAttrTypeIdToNewAttrType.put(oldAttribute.getAttributeType().getTypeID(), newCustomType); return newCustomType; @@ -975,39 +998,38 @@ public class PortableCaseReportModule implements ReportModule { /** * Top level method to copy a content object to the portable case. - * - * @param content The content object to copy + * + * @param content The content object to copy * @param progressPanel The progress panel - * + * * @return The object ID of the copied content in the portable case - * - * @throws TskCoreException + * + * @throws TskCoreException */ @NbBundle.Messages({ "# {0} - File name", - "PortableCaseReportModule.copyContentToPortableCase.copyingFile=Copying file {0}", - }) + "PortableCaseReportModule.copyContentToPortableCase.copyingFile=Copying file {0}",}) private long copyContentToPortableCase(Content content, ReportProgressPanel progressPanel) throws TskCoreException { progressPanel.updateStatusLabel(Bundle.PortableCaseReportModule_copyContentToPortableCase_copyingFile(content.getUniquePath())); return copyContent(content); } - + /** * Returns the object ID for the given content object in the portable case. - * + * * @param content The content object to copy into the portable case - * + * * @return the new object ID for this content - * - * @throws TskCoreException + * + * @throws TskCoreException */ private long copyContent(Content content) throws TskCoreException { - + // Check if we've already copied this content if (oldIdToNewContent.containsKey(content.getId())) { return oldIdToNewContent.get(content.getId()).getId(); } - + // Otherwise: // - Make parent of this object (if applicable) // - Copy this content @@ -1015,42 +1037,42 @@ public class PortableCaseReportModule implements ReportModule { if (content.getParent() != null) { parentId = copyContent(content.getParent()); } - + Content newContent; if (content instanceof BlackboardArtifact) { - BlackboardArtifact artifactToCopy = (BlackboardArtifact)content; + BlackboardArtifact artifactToCopy = (BlackboardArtifact) content; newContent = copyArtifact(parentId, artifactToCopy); } else { CaseDbTransaction trans = portableSkCase.beginTransaction(); try { if (content instanceof Image) { - Image image = (Image)content; - newContent = portableSkCase.addImage(image.getType(), image.getSsize(), image.getSize(), image.getName(), + Image image = (Image) content; + newContent = portableSkCase.addImage(image.getType(), image.getSsize(), image.getSize(), image.getName(), new ArrayList<>(), image.getTimeZone(), image.getMd5(), image.getSha1(), image.getSha256(), image.getDeviceId(), trans); } else if (content instanceof VolumeSystem) { - VolumeSystem vs = (VolumeSystem)content; + VolumeSystem vs = (VolumeSystem) content; newContent = portableSkCase.addVolumeSystem(parentId, vs.getType(), vs.getOffset(), vs.getBlockSize(), trans); } else if (content instanceof Volume) { - Volume vs = (Volume)content; - newContent = portableSkCase.addVolume(parentId, vs.getAddr(), vs.getStart(), vs.getLength(), + Volume vs = (Volume) content; + newContent = portableSkCase.addVolume(parentId, vs.getAddr(), vs.getStart(), vs.getLength(), vs.getDescription(), vs.getFlags(), trans); } else if (content instanceof Pool) { - Pool pool = (Pool)content; + Pool pool = (Pool) content; newContent = portableSkCase.addPool(parentId, pool.getType(), trans); } else if (content instanceof FileSystem) { - FileSystem fs = (FileSystem)content; - newContent = portableSkCase.addFileSystem(parentId, fs.getImageOffset(), fs.getFsType(), fs.getBlock_size(), - fs.getBlock_count(), fs.getRoot_inum(), fs.getFirst_inum(), fs.getLastInum(), + FileSystem fs = (FileSystem) content; + newContent = portableSkCase.addFileSystem(parentId, fs.getImageOffset(), fs.getFsType(), fs.getBlock_size(), + fs.getBlock_count(), fs.getRoot_inum(), fs.getFirst_inum(), fs.getLastInum(), fs.getName(), trans); } else if (content instanceof BlackboardArtifact) { - BlackboardArtifact artifactToCopy = (BlackboardArtifact)content; + BlackboardArtifact artifactToCopy = (BlackboardArtifact) content; newContent = copyArtifact(parentId, artifactToCopy); } else if (content instanceof AbstractFile) { - AbstractFile abstractFile = (AbstractFile)content; - + AbstractFile abstractFile = (AbstractFile) content; + if (abstractFile instanceof LocalFilesDataSource) { - LocalFilesDataSource localFilesDS = (LocalFilesDataSource)abstractFile; - newContent = portableSkCase.addLocalFilesDataSource(localFilesDS.getDeviceId(), localFilesDS.getName(), localFilesDS.getTimeZone(), trans); + LocalFilesDataSource localFilesDS = (LocalFilesDataSource) abstractFile; + newContent = portableSkCase.addLocalFilesDataSource(localFilesDS.getDeviceId(), localFilesDS.getName(), localFilesDS.getTimeZone(), trans); } else { if (abstractFile.isDir()) { newContent = portableSkCase.addLocalDirectory(parentId, abstractFile.getName(), trans); @@ -1065,21 +1087,21 @@ public class PortableCaseReportModule implements ReportModule { // Get the new parent object in the portable case database Content oldParent = abstractFile.getParent(); - if (! oldIdToNewContent.containsKey(oldParent.getId())) { + if (!oldIdToNewContent.containsKey(oldParent.getId())) { throw new TskCoreException("Parent of file with ID " + abstractFile.getId() + " has not been created"); // NON-NLS } Content newParent = oldIdToNewContent.get(oldParent.getId()); // Construct the relative path to the copied file - String relativePath = FILE_FOLDER_NAME + File.separator + exportSubFolder + File.separator + fileName; + String relativePath = FILE_FOLDER_NAME + File.separator + exportSubFolder + File.separator + fileName; newContent = portableSkCase.addLocalFile(abstractFile.getName(), relativePath, abstractFile.getSize(), abstractFile.getCtime(), abstractFile.getCrtime(), abstractFile.getAtime(), abstractFile.getMtime(), abstractFile.getMd5Hash(), abstractFile.getKnown(), abstractFile.getMIMEType(), - true, TskData.EncodingType.NONE, + true, TskData.EncodingType.NONE, newParent, trans); } catch (IOException ex) { - throw new TskCoreException("Error copying file " + abstractFile.getName() + " with original obj ID " + throw new TskCoreException("Error copying file " + abstractFile.getName() + " with original obj ID " + abstractFile.getId(), ex); // NON-NLS } } @@ -1088,38 +1110,38 @@ public class PortableCaseReportModule implements ReportModule { throw new TskCoreException("Trying to copy unexpected Content type " + content.getClass().getName()); // NON-NLS } trans.commit(); - } catch (TskCoreException ex) { + } catch (TskCoreException ex) { trans.rollback(); - throw(ex); + throw (ex); } } - + // Save the new object oldIdToNewContent.put(content.getId(), newContent); newIdToContent.put(newContent.getId(), newContent); return oldIdToNewContent.get(content.getId()).getId(); } - + /** * Return the subfolder name for this file based on MIME type - * + * * @param abstractFile the file - * - * @return the name of the appropriate subfolder for this file type + * + * @return the name of the appropriate subfolder for this file type */ private String getExportSubfolder(AbstractFile abstractFile) { if (abstractFile.getMIMEType() == null || abstractFile.getMIMEType().isEmpty()) { return UNKNOWN_FILE_TYPE_FOLDER; } - - for (FileTypeCategory cat:FILE_TYPE_CATEGORIES) { + + for (FileTypeCategory cat : FILE_TYPE_CATEGORIES) { if (cat.getMediaTypes().contains(abstractFile.getMIMEType())) { return cat.getDisplayName(); } } return UNKNOWN_FILE_TYPE_FOLDER; } - + /** * Clear out the maps and other fields and close the database connections. */ @@ -1132,12 +1154,12 @@ public class PortableCaseReportModule implements ReportModule { oldArtifactIdToNewArtifact.clear(); closePortableCaseDatabase(); - + currentCase = null; caseFolder = null; copiedFilesFolder = null; } - + /** * Close the portable case */ @@ -1153,15 +1175,14 @@ public class PortableCaseReportModule implements ReportModule { configPanel = new CreatePortableCasePanel(); return configPanel; } */ - private class StoreMaxIdCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback { private final String tableName; - + StoreMaxIdCallback(String tableName) { this.tableName = tableName; } - + @Override public void process(ResultSet rs) { @@ -1177,60 +1198,59 @@ public class PortableCaseReportModule implements ReportModule { } catch (TskCoreException ex) { logger.log(Level.WARNING, "Unable to save maximum ID from result set", ex); // NON-NLS } - + } } catch (SQLException ex) { logger.log(Level.WARNING, "Failed to get maximum ID from result set", ex); // NON-NLS } } } - + @NbBundle.Messages({ "PortableCaseReportModule.compressCase.errorFinding7zip=Could not locate 7-Zip executable", "# {0} - Temp folder path", "PortableCaseReportModule.compressCase.errorCreatingTempFolder=Could not create temporary folder {0}", "PortableCaseReportModule.compressCase.errorCompressingCase=Error compressing case", - "PortableCaseReportModule.compressCase.canceled=Compression canceled by user", - }) + "PortableCaseReportModule.compressCase.canceled=Compression canceled by user",}) private boolean compressCase(ReportProgressPanel progressPanel) { - + // Close the portable case database (we still need some of the variables that would be cleared by cleanup()) closePortableCaseDatabase(); - + // Make a temporary folder for the compressed case File tempZipFolder = Paths.get(currentCase.getTempDirectory(), "portableCase" + System.currentTimeMillis()).toFile(); // NON-NLS - if (! tempZipFolder.mkdir()) { - handleError("Error creating temporary folder " + tempZipFolder.toString(), + if (!tempZipFolder.mkdir()) { + handleError("Error creating temporary folder " + tempZipFolder.toString(), Bundle.PortableCaseReportModule_compressCase_errorCreatingTempFolder(tempZipFolder.toString()), null, progressPanel); // NON-NLS return false; } - + // Find 7-Zip File sevenZipExe = locate7ZipExecutable(); if (sevenZipExe == null) { handleError("Error finding 7-Zip exectuable", Bundle.PortableCaseReportModule_compressCase_errorFinding7zip(), null, progressPanel); // NON-NLS return false; } - + // Create the chunk option String chunkOption = ""; if (settings.getChunkSize() != PortableCaseReportModuleSettings.ChunkSize.NONE) { chunkOption = "-v" + settings.getChunkSize().getSevenZipParam(); } - + File zipFile = Paths.get(tempZipFolder.getAbsolutePath(), caseName + ".zip").toFile(); // NON-NLS ProcessBuilder procBuilder = new ProcessBuilder(); procBuilder.command( sevenZipExe.getAbsolutePath(), - "a", // Add to archive + "a", // Add to archive zipFile.getAbsolutePath(), caseFolder.getAbsolutePath(), chunkOption ); - + try { Process process = procBuilder.start(); - + while (process.isAlive()) { if (progressPanel.getStatus() == ReportProgressPanel.ReportStatus.CANCELED) { process.destroy(); @@ -1248,7 +1268,7 @@ public class PortableCaseReportModule implements ReportModule { sb.append(line).append(System.getProperty("line.separator")); // NON-NLS } } - + handleError("Error compressing case\n7-Zip output: " + sb.toString(), Bundle.PortableCaseReportModule_compressCase_errorCompressingCase(), null, progressPanel); // NON-NLS return false; } @@ -1256,7 +1276,7 @@ public class PortableCaseReportModule implements ReportModule { handleError("Error compressing case", Bundle.PortableCaseReportModule_compressCase_errorCompressingCase(), ex, progressPanel); // NON-NLS return false; } - + // Delete everything in the case folder then copy over the compressed file(s) try { FileUtils.cleanDirectory(caseFolder); @@ -1266,10 +1286,10 @@ public class PortableCaseReportModule implements ReportModule { handleError("Error compressing case", Bundle.PortableCaseReportModule_compressCase_errorCompressingCase(), ex, progressPanel); // NON-NLS return false; } - + return true; } - + /** * Locate the 7-Zip executable from the release folder. * @@ -1292,7 +1312,7 @@ public class PortableCaseReportModule implements ReportModule { return exeFile; } - + /** * Processes the result sets from the interesting item set name query. */ @@ -1300,7 +1320,7 @@ public class PortableCaseReportModule implements ReportModule { private static final java.util.logging.Logger logger = java.util.logging.Logger.getLogger(GetInterestingItemSetNamesCallback.class.getName()); private final Map setCounts = new HashMap<>(); - + @Override public void process(ResultSet rs) { try { @@ -1310,7 +1330,7 @@ public class PortableCaseReportModule implements ReportModule { String setName = rs.getString("set_name"); // NON-NLS setCounts.put(setName, setCount); - + } catch (SQLException ex) { logger.log(Level.WARNING, "Unable to get data_source_obj_id or value from result set", ex); // NON-NLS } @@ -1318,11 +1338,11 @@ public class PortableCaseReportModule implements ReportModule { } catch (SQLException ex) { logger.log(Level.WARNING, "Failed to get next result for values by datasource", ex); // NON-NLS } - } - + } + /** * Gets the counts for each interesting items set - * + * * @return A map from each set name to the number of items in it */ public Map getSetCountMap() { From d3043afefdf49c1e434a1b71dac31a91ebfb5148 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 14 Jul 2020 11:23:00 -0400 Subject: [PATCH 014/102] fix in readme --- release_scripts/localization_scripts/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release_scripts/localization_scripts/README.md b/release_scripts/localization_scripts/README.md index f2db66fcc8..0b738f8f38 100644 --- a/release_scripts/localization_scripts/README.md +++ b/release_scripts/localization_scripts/README.md @@ -9,9 +9,9 @@ All of these scripts provide more details on usage by calling the script with `- ## Basic Localization Update Workflow -1. Call `python3 diffscript.py -l ` to generate a csv file containing differences in properties file values from the language's previous commit to the `HEAD` commit. The language identifier should be the abbreviated identifier used for the bundle (i.e. 'ja' for Japanese). +1. Call `python3 diffscript.py -l ` to generate a csv file containing differences in properties file values from the language's previous commit to the `HEAD` commit. The language identifier should be the abbreviated identifier used for the bundle (i.e. 'ja' for Japanese). 2. Update csv file with translations -3. Call `python3 updatepropsscript.py -l ` to update properties files based on the newly generated csv file. The csv file should be formatted such that the columns are bundle relative path, property files key, translated value and commit id for the latest commit id for which these changes represent. The commit id only needs to be in the header row. +3. Call `python3 updatepropsscript.py -l ` to update properties files based on the newly generated csv file. The csv file should be formatted such that the columns are bundle relative path, property files key, translated value and commit id for the latest commit id for which these changes represent. The commit id only needs to be in the header row. ## Localization Generation for the First Time First-time updates should follow a similar procedure except that instead of calling `diffscript.py`, call `python3 allbundlesscript ` to generate a csv file with relative paths of bundle files, property file keys, property file values. \ No newline at end of file From 017f9b3b344c81d21dc402f9e2735c1c0c394925 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 14 Jul 2020 11:35:28 -0400 Subject: [PATCH 015/102] Updated comments and made a bug fix --- .../modules/caseuco/CaseUcoReportModule.java | 75 ++++++++++++------- .../PortableCaseReportModule.java | 2 - 2 files changed, 46 insertions(+), 31 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java b/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java index 2da96ac34f..2731515ca9 100755 --- a/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java +++ b/Core/src/org/sleuthkit/autopsy/report/modules/caseuco/CaseUcoReportModule.java @@ -59,8 +59,8 @@ import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.blackboardutils.attributes.BlackboardJsonAttrUtil; /** - * CaseUcoReportModule generates a report in CASE-UCO format. This module will - * write all files and data sources to the report. + * Exports an Autopsy case to a CASE-UCO report file. This module will write all + * files and artifacts from the selected data sources. */ public final class CaseUcoReportModule implements GeneralReportModule { @@ -156,7 +156,7 @@ public final class CaseUcoReportModule implements GeneralReportModule { return; } - Case caseObj = Case.getCurrentCaseThrows(); + Case currentCase = Case.getCurrentCaseThrows(); Path caseJsonReportFile = reportDirectory.resolve(REPORT_FILE_NAME + "." + EXTENSION); @@ -168,28 +168,21 @@ public final class CaseUcoReportModule implements GeneralReportModule { reportWriter.name("@graph"); reportWriter.beginArray(); - //First write the Case to the report file. - CaseUcoExporter exporter = new CaseUcoExporter(caseObj.getSleuthkitCase()); + CaseUcoExporter exporter = new CaseUcoExporter(currentCase.getSleuthkitCase()); for (JsonElement element : exporter.exportSleuthkitCase()) { gson.toJson(element, reportWriter); } - // Prune the data sources so that we only report on what was selected. - List dataSources = caseObj.getSleuthkitCase().getDataSources().stream() - .filter((dataSource) -> { - if (settings.getSelectedDataSources() == null) { - // Assume all data sources if list is null. - return true; - } - return settings.getSelectedDataSources().contains(dataSource.getId()); - }) - .collect(Collectors.toList()); + // Get a list of selected data sources to process. + List dataSources = getSelectedDataSources(currentCase, settings); progressPanel.setIndeterminate(false); progressPanel.setMaximumProgress(dataSources.size()); progressPanel.start(); - //Then search each data source for file content. + // First stage of reporting is for files and data sources. + // Iterate through each data source and dump all files contained + // in that data source. for (int i = 0; i < dataSources.size(); i++) { DataSource dataSource = dataSources.get(i); progressPanel.updateStatusLabel(String.format( @@ -204,17 +197,26 @@ public final class CaseUcoReportModule implements GeneralReportModule { progressPanel.setProgress(i + 1); } - // Write all standard artifacts to the report. - for (ARTIFACT_TYPE artType : caseObj.getSleuthkitCase().getBlackboardArtifactTypesInUse()) { - for (BlackboardArtifact artifact : caseObj.getSleuthkitCase().getBlackboardArtifacts(artType)) { - try { - for (JsonElement element : exporter.exportBlackboardArtifact(artifact)) { - gson.toJson(element, reportWriter); + // Second stage of reporting handles artifacts. + Set dataSourceIds = dataSources.stream() + .map((datasource) -> datasource.getId()) + .collect(Collectors.toSet()); + + // Write all standard artifacts that are contained within the + // selected data sources. + for (ARTIFACT_TYPE artType : currentCase.getSleuthkitCase().getBlackboardArtifactTypesInUse()) { + for (BlackboardArtifact artifact : currentCase.getSleuthkitCase().getBlackboardArtifacts(artType)) { + if (dataSourceIds.contains(artifact.getDataSource().getId())) { + + try { + for (JsonElement element : exporter.exportBlackboardArtifact(artifact)) { + gson.toJson(element, reportWriter); + } + } catch (ContentNotExportableException | BlackboardJsonAttrUtil.InvalidJsonException ex) { + logger.log(Level.WARNING, String.format("Unable to export blackboard artifact (id: %d) to CASE/UCO. " + + "The artifact type is either not supported or the artifact instance does not have any " + + "exportable attributes.", artifact.getId())); } - } catch (ContentNotExportableException | BlackboardJsonAttrUtil.InvalidJsonException ex) { - logger.log(Level.WARNING, String.format("Unable to export blackboard artifact (id: %d) to CASE/UCO. " - + "The artifact type is either not supported or the artifact instance does not have any " - + "exportable attributes.", artifact.getId())); } } } @@ -223,7 +225,7 @@ public final class CaseUcoReportModule implements GeneralReportModule { reportWriter.endObject(); } - caseObj.addReport(caseJsonReportFile.toString(), + currentCase.addReport(caseJsonReportFile.toString(), Bundle.CaseUcoReportModule_srcModuleName(), REPORT_FILE_NAME); progressPanel.complete(ReportProgressPanel.ReportStatus.COMPLETE); @@ -244,6 +246,21 @@ public final class CaseUcoReportModule implements GeneralReportModule { progressPanel.complete(ReportProgressPanel.ReportStatus.COMPLETE); } + /** + * Get the selected data sources from the settings instance. + */ + private List getSelectedDataSources(Case currentCase, GeneralReportSettings settings) throws TskCoreException { + return currentCase.getSleuthkitCase().getDataSources().stream() + .filter((dataSource) -> { + if (settings.getSelectedDataSources() == null) { + // Assume all data sources if list is null. + return true; + } + return settings.getSelectedDataSources().contains(dataSource.getId()); + }) + .collect(Collectors.toList()); + } + /** * Warn the user if ingest is still ongoing. */ @@ -268,7 +285,7 @@ public final class CaseUcoReportModule implements GeneralReportModule { if (current instanceof AbstractFile) { AbstractFile file = (AbstractFile) (current); if (SUPPORTED_TYPES.contains(file.getMetaType().getValue())) { - + for (JsonElement element : exporter.exportAbstractFile(file)) { gson.toJson(element, reportWriter); } @@ -280,4 +297,4 @@ public final class CaseUcoReportModule implements GeneralReportModule { } } } -} \ No newline at end of file +} diff --git a/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java b/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java index b871d4bf84..b02b95aaa6 100644 --- a/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java +++ b/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java @@ -481,7 +481,6 @@ public class PortableCaseReportModule implements ReportModule { reportWriter.name("@graph"); reportWriter.beginArray(); - //Acquire references to querying files. String caseTempDirectory = currentCase.getTempDirectory(); SleuthkitCase skCase = currentCase.getSleuthkitCase(); TagsManager tagsManager = currentCase.getServices().getTagsManager(); @@ -492,7 +491,6 @@ public class PortableCaseReportModule implements ReportModule { FileUtils.deleteDirectory(tmpDir.toFile()); Files.createDirectory(tmpDir); - //First export the Sleuthkit Case instance. CaseUcoExporter exporter = new CaseUcoExporter(currentCase.getSleuthkitCase()); for (JsonElement element : exporter.exportSleuthkitCase()) { gson.toJson(element, reportWriter); From 9e9b702c31a2631fc18f2431ea1eb1c21c17814b Mon Sep 17 00:00:00 2001 From: apriestman Date: Wed, 15 Jul 2020 07:24:19 -0400 Subject: [PATCH 016/102] Change dist-PostgreSQL targets to dist --- BUILDING.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/BUILDING.txt b/BUILDING.txt index 570cadbf87..a77d6c8add 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -37,16 +37,16 @@ to the root 64-bit JRE directory. 2) Get Sleuth Kit Setup 2a) Download and build a Release version of Sleuth Kit (TSK) 4.0. See win32\BUILDING.txt in the TSK package for more information. You need to - build the tsk_jni project. Select the Release_PostgreSQL Win32 or x64 target, + build the tsk_jni project. Select the Release Win32 or x64 target, depending upon your target build. You can use a released version or download the latest from github: - git://github.com/sleuthkit/sleuthkit.git -2b) Build the TSK JAR file by typing 'ant dist-PostgreSQL' in +2b) Build the TSK JAR file by typing 'ant dist' in bindings/java in the TSK source code folder from a command line. Note it is case sensitive. You can also add the code to a NetBeans project and build - it from there, selecting the dist-PostgreSQL target. + it from there, selecting the dist target. 2c) Set TSK_HOME environment variable to the root directory of TSK @@ -103,7 +103,7 @@ the build process. - The Sleuth Kit Java datamodel JAR file has native JNI libraries that are copied into it. These JNI libraries have dependencies on -libewf, zlib, libpq, libintl-8, libeay32, and ssleay32 DLL files. On non-Windows +libewf, zlib, libintl-8, libeay32, and ssleay32 DLL files. On non-Windows platforms, the JNI library also has a dependency on libtsk (on Windows, it is compiled into libtsk_jni). From 6ad41d2ae1ba24d7d02c4860357cd6b117083821 Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Wed, 15 Jul 2020 10:04:57 -0400 Subject: [PATCH 017/102] 6611: Fix background of context content viewer --- .../contextviewer/ContextSourcePanel.form | 3 --- .../contextviewer/ContextSourcePanel.java | 1 - .../contextviewer/ContextUsagePanel.form | 3 --- .../contextviewer/ContextUsagePanel.java | 1 - .../contextviewer/ContextViewer.form | 15 --------------- .../contextviewer/ContextViewer.java | 6 ------ 6 files changed, 29 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextSourcePanel.form b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextSourcePanel.form index c57a0990a8..76a335dbc7 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextSourcePanel.form +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextSourcePanel.form @@ -2,9 +2,6 @@
- - - diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextSourcePanel.java b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextSourcePanel.java index 284a66cf04..e0935d8ae4 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextSourcePanel.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextSourcePanel.java @@ -65,7 +65,6 @@ public final class ContextSourcePanel extends javax.swing.JPanel { jSourceNameLabel = new javax.swing.JLabel(); jSourceTextLabel = new javax.swing.JLabel(); - setBackground(new java.awt.Color(255, 255, 255)); setPreferredSize(new java.awt.Dimension(495, 75)); org.openide.awt.Mnemonics.setLocalizedText(jSourceGoToResultButton, org.openide.util.NbBundle.getMessage(ContextSourcePanel.class, "ContextSourcePanel.jSourceGoToResultButton.text")); // NOI18N diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextUsagePanel.form b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextUsagePanel.form index a3e2a90f84..c618b3f342 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextUsagePanel.form +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextUsagePanel.form @@ -2,9 +2,6 @@ - - - diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextUsagePanel.java b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextUsagePanel.java index de87d7a5e5..f26618ad50 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextUsagePanel.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextUsagePanel.java @@ -65,7 +65,6 @@ public final class ContextUsagePanel extends javax.swing.JPanel { jUsageNameLabel = new javax.swing.JLabel(); jUsageTextLabel = new javax.swing.JLabel(); - setBackground(new java.awt.Color(255, 255, 255)); setPreferredSize(new java.awt.Dimension(495, 75)); org.openide.awt.Mnemonics.setLocalizedText(jUsageGoToResultButton, org.openide.util.NbBundle.getMessage(ContextUsagePanel.class, "ContextUsagePanel.jUsageGoToResultButton.text")); // NOI18N diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextViewer.form b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextViewer.form index 01c484ba9e..22e2953851 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextViewer.form +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextViewer.form @@ -3,11 +3,6 @@ - - - - - @@ -49,11 +44,6 @@ - - - - - @@ -95,11 +85,6 @@ - - - - - diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextViewer.java index f7f1f6559e..ff1d02ccd2 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextViewer.java @@ -89,8 +89,6 @@ public final class ContextViewer extends javax.swing.JPanel implements DataConte javax.swing.JLabel jUnknownLabel = new javax.swing.JLabel(); jScrollPane = new javax.swing.JScrollPane(); - jSourcePanel.setBackground(javax.swing.UIManager.getDefaults().getColor("window")); - jSourceLabel.setFont(jSourceLabel.getFont().deriveFont(jSourceLabel.getFont().getStyle() | java.awt.Font.BOLD, jSourceLabel.getFont().getSize()+1)); org.openide.awt.Mnemonics.setLocalizedText(jSourceLabel, org.openide.util.NbBundle.getMessage(ContextViewer.class, "ContextViewer.jSourceLabel.text")); // NOI18N @@ -111,8 +109,6 @@ public final class ContextViewer extends javax.swing.JPanel implements DataConte .addGap(2, 2, 2)) ); - jUsagePanel.setBackground(javax.swing.UIManager.getDefaults().getColor("window")); - jUsageLabel.setFont(jUsageLabel.getFont().deriveFont(jUsageLabel.getFont().getStyle() | java.awt.Font.BOLD, jUsageLabel.getFont().getSize()+1)); org.openide.awt.Mnemonics.setLocalizedText(jUsageLabel, org.openide.util.NbBundle.getMessage(ContextViewer.class, "ContextViewer.jUsageLabel.text")); // NOI18N @@ -133,8 +129,6 @@ public final class ContextViewer extends javax.swing.JPanel implements DataConte .addGap(2, 2, 2)) ); - jUnknownPanel.setBackground(new java.awt.Color(255, 255, 255)); - org.openide.awt.Mnemonics.setLocalizedText(jUnknownLabel, org.openide.util.NbBundle.getMessage(ContextViewer.class, "ContextViewer.jUnknownLabel.text")); // NOI18N javax.swing.GroupLayout jUnknownPanelLayout = new javax.swing.GroupLayout(jUnknownPanel); From 93dcc2199da9218e613ada84870f707d573d6c44 Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Wed, 15 Jul 2020 10:10:35 -0400 Subject: [PATCH 018/102] Revert "6611: Fix background of context content viewer" This reverts commit 6ad41d2ae1ba24d7d02c4860357cd6b117083821. --- .../contextviewer/ContextSourcePanel.form | 3 +++ .../contextviewer/ContextSourcePanel.java | 1 + .../contextviewer/ContextUsagePanel.form | 3 +++ .../contextviewer/ContextUsagePanel.java | 1 + .../contextviewer/ContextViewer.form | 15 +++++++++++++++ .../contextviewer/ContextViewer.java | 6 ++++++ 6 files changed, 29 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextSourcePanel.form b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextSourcePanel.form index 76a335dbc7..c57a0990a8 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextSourcePanel.form +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextSourcePanel.form @@ -2,6 +2,9 @@ + + + diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextSourcePanel.java b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextSourcePanel.java index e0935d8ae4..284a66cf04 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextSourcePanel.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextSourcePanel.java @@ -65,6 +65,7 @@ public final class ContextSourcePanel extends javax.swing.JPanel { jSourceNameLabel = new javax.swing.JLabel(); jSourceTextLabel = new javax.swing.JLabel(); + setBackground(new java.awt.Color(255, 255, 255)); setPreferredSize(new java.awt.Dimension(495, 75)); org.openide.awt.Mnemonics.setLocalizedText(jSourceGoToResultButton, org.openide.util.NbBundle.getMessage(ContextSourcePanel.class, "ContextSourcePanel.jSourceGoToResultButton.text")); // NOI18N diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextUsagePanel.form b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextUsagePanel.form index c618b3f342..a3e2a90f84 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextUsagePanel.form +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextUsagePanel.form @@ -2,6 +2,9 @@ + + + diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextUsagePanel.java b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextUsagePanel.java index f26618ad50..de87d7a5e5 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextUsagePanel.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextUsagePanel.java @@ -65,6 +65,7 @@ public final class ContextUsagePanel extends javax.swing.JPanel { jUsageNameLabel = new javax.swing.JLabel(); jUsageTextLabel = new javax.swing.JLabel(); + setBackground(new java.awt.Color(255, 255, 255)); setPreferredSize(new java.awt.Dimension(495, 75)); org.openide.awt.Mnemonics.setLocalizedText(jUsageGoToResultButton, org.openide.util.NbBundle.getMessage(ContextUsagePanel.class, "ContextUsagePanel.jUsageGoToResultButton.text")); // NOI18N diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextViewer.form b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextViewer.form index 22e2953851..01c484ba9e 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextViewer.form +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextViewer.form @@ -3,6 +3,11 @@ + + + + + @@ -44,6 +49,11 @@ + + + + + @@ -85,6 +95,11 @@ + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextViewer.java index ff1d02ccd2..f7f1f6559e 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/contextviewer/ContextViewer.java @@ -89,6 +89,8 @@ public final class ContextViewer extends javax.swing.JPanel implements DataConte javax.swing.JLabel jUnknownLabel = new javax.swing.JLabel(); jScrollPane = new javax.swing.JScrollPane(); + jSourcePanel.setBackground(javax.swing.UIManager.getDefaults().getColor("window")); + jSourceLabel.setFont(jSourceLabel.getFont().deriveFont(jSourceLabel.getFont().getStyle() | java.awt.Font.BOLD, jSourceLabel.getFont().getSize()+1)); org.openide.awt.Mnemonics.setLocalizedText(jSourceLabel, org.openide.util.NbBundle.getMessage(ContextViewer.class, "ContextViewer.jSourceLabel.text")); // NOI18N @@ -109,6 +111,8 @@ public final class ContextViewer extends javax.swing.JPanel implements DataConte .addGap(2, 2, 2)) ); + jUsagePanel.setBackground(javax.swing.UIManager.getDefaults().getColor("window")); + jUsageLabel.setFont(jUsageLabel.getFont().deriveFont(jUsageLabel.getFont().getStyle() | java.awt.Font.BOLD, jUsageLabel.getFont().getSize()+1)); org.openide.awt.Mnemonics.setLocalizedText(jUsageLabel, org.openide.util.NbBundle.getMessage(ContextViewer.class, "ContextViewer.jUsageLabel.text")); // NOI18N @@ -129,6 +133,8 @@ public final class ContextViewer extends javax.swing.JPanel implements DataConte .addGap(2, 2, 2)) ); + jUnknownPanel.setBackground(new java.awt.Color(255, 255, 255)); + org.openide.awt.Mnemonics.setLocalizedText(jUnknownLabel, org.openide.util.NbBundle.getMessage(ContextViewer.class, "ContextViewer.jUnknownLabel.text")); // NOI18N javax.swing.GroupLayout jUnknownPanelLayout = new javax.swing.GroupLayout(jUnknownPanel); From d57e0df83178d32be54e982a24d6540272d3b58a Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 15 Jul 2020 16:32:31 -0400 Subject: [PATCH 019/102] update for central repo upgrade notifications --- .../datamodel/CentralRepoDbUtil.java | 4 +--- .../optionspanel/EamDbSettingsDialog.java | 18 ++++++++++++++++++ .../persona/PersonasTopComponent.java | 2 +- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDbUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDbUtil.java index 6ba23b65b5..5105aed2e9 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDbUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDbUtil.java @@ -262,9 +262,7 @@ public class CentralRepoDbUtil { * used */ public static void setUseCentralRepo(boolean centralRepoCheckBoxIsSelected) { - if (!centralRepoCheckBoxIsSelected) { - closePersonasTopComponent(); - } + closePersonasTopComponent(); ModuleSettings.setConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_USE_KEY, Boolean.toString(centralRepoCheckBoxIsSelected)); } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java index 675e7c8807..f30c402513 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java @@ -43,6 +43,7 @@ import javax.swing.event.DocumentListener; import javax.swing.filechooser.FileFilter; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; +import org.openide.windows.TopComponent; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoDbChoice; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoDbManager; @@ -660,6 +661,8 @@ public class EamDbSettingsDialog extends JDialog { * found. */ private static boolean testStatusAndCreate(Component parent, CentralRepoDbManager manager, EamDbSettingsDialog dialog) { + closePersonasTopComponent(); + parent.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); manager.testStatus(); @@ -690,6 +693,21 @@ public class EamDbSettingsDialog extends JDialog { parent.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); return true; } + + + + /** + * Closes Personas top component if it exists. + */ + private static void closePersonasTopComponent() { + SwingUtilities.invokeLater(() -> { + TopComponent personasWindow = WindowManager.getDefault().findTopComponent("PersonasTopComponent"); + if (personasWindow != null && personasWindow.isOpened()) { + personasWindow.close(); + } + }); + } + /** * This method returns if changes to the central repository configuration diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/persona/PersonasTopComponent.java b/Core/src/org/sleuthkit/autopsy/centralrepository/persona/PersonasTopComponent.java index d38b379078..730c4bed47 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/persona/PersonasTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/persona/PersonasTopComponent.java @@ -276,7 +276,7 @@ public final class PersonasTopComponent extends TopComponent { } @Messages({ - "PersonasTopComponent_search_exception_Title=Search failure", + "PersonasTopComponent_search_exception_Title=There was a failure during the search. Try opening a case to fully initialize the central repository database.", "PersonasTopComponent_search_exception_msg=Failed to search personas.", "PersonasTopComponent_noCR_msg=Central Repository is not enabled.",}) private void executeSearch() { From 1e749e5e53eb9f1cc182d9fb90b52b4e40d3267e Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 16 Jul 2020 08:31:17 -0400 Subject: [PATCH 020/102] used component listener so that only one event fired per open --- .../persona/PersonasTopComponent.java | 36 +++++++------------ 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/persona/PersonasTopComponent.java b/Core/src/org/sleuthkit/autopsy/centralrepository/persona/PersonasTopComponent.java index 730c4bed47..e051529f11 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/persona/PersonasTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/persona/PersonasTopComponent.java @@ -20,6 +20,8 @@ package org.sleuthkit.autopsy.centralrepository.persona; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; +import java.awt.event.ComponentAdapter; +import java.awt.event.ComponentEvent; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -60,28 +62,6 @@ public final class PersonasTopComponent extends TopComponent { private List currentResults = null; private Persona selectedPersona = null; - /** - * Listens for when this component will be rendered and executes a search to - * update gui when it is displayed. - */ - private final AncestorListener onAddListener = new AncestorListener() { - @Override - public void ancestorAdded(AncestorEvent event) { - resetSearchControls(); - setKeywordSearchEnabled(false, true); - } - - @Override - public void ancestorRemoved(AncestorEvent event) { - //Empty - } - - @Override - public void ancestorMoved(AncestorEvent event) { - //Empty - } - }; - @Messages({ "PersonasTopComponent_Name=Personas", "PersonasTopComponent_delete_exception_Title=Delete failure", @@ -165,7 +145,17 @@ public final class PersonasTopComponent extends TopComponent { } }); - addAncestorListener(onAddListener); + /** + * Listens for when this component will be rendered and executes a + * search to update gui when it is displayed. + */ + addComponentListener(new ComponentAdapter() { + @Override + public void componentShown(ComponentEvent e) { + resetSearchControls(); + setKeywordSearchEnabled(false, true); + } + }); } /** From 048b809c2cd2ca86f6287c9025392fba635e9189 Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Thu, 16 Jul 2020 11:29:46 -0400 Subject: [PATCH 021/102] 6619: incorrect missing account --- .../datamodel/CentralRepository.java | 13 +++++++++++ .../datamodel/RdbmsCentralRepo.java | 3 ++- .../ContactArtifactViewer.java | 23 ++++++++----------- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepository.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepository.java index bdae5a727b..a28b57e30a 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepository.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepository.java @@ -884,5 +884,18 @@ public interface CentralRepository { * @throws CentralRepoException */ CentralRepoAccount getOrCreateAccount(CentralRepoAccount.CentralRepoAccountType crAccountType, String accountUniqueID) throws CentralRepoException; + + /** + * Gets an account from the accounts table matching the given type/ID, if + * one exists. + * + * @param crAccountType CR account type to look for or create + * @param accountUniqueID type specific unique account id + * + * @return CR account, if found, null otherwise. + * + * @throws CentralRepoException + */ + CentralRepoAccount getAccount(CentralRepoAccount.CentralRepoAccountType crAccountType, String accountUniqueID) throws CentralRepoException; } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java index d3195a6d3d..70c98f4eab 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/RdbmsCentralRepo.java @@ -1179,7 +1179,8 @@ abstract class RdbmsCentralRepo implements CentralRepository { * * @throws CentralRepoException */ - private CentralRepoAccount getAccount(CentralRepoAccountType crAccountType, String accountUniqueID) throws CentralRepoException { + @Override + public CentralRepoAccount getAccount(CentralRepoAccountType crAccountType, String accountUniqueID) throws CentralRepoException { CentralRepoAccount crAccount = accountsCache.getIfPresent(Pair.of(crAccountType, accountUniqueID)); if (crAccount == null) { diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/artifactviewers/ContactArtifactViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/artifactviewers/ContactArtifactViewer.java index 734a7ec180..95da68ca10 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/artifactviewers/ContactArtifactViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/artifactviewers/ContactArtifactViewer.java @@ -633,21 +633,18 @@ public class ContactArtifactViewer extends javax.swing.JPanel implements Artifac return new HashMap<>(); } + // make a list of all unique accounts for this contact + if (!account.getAccountType().equals(Account.Type.DEVICE)) { + CentralRepoAccount.CentralRepoAccountType crAccountType = CentralRepository.getInstance().getAccountTypeByName(account.getAccountType().getTypeName()); + CentralRepoAccount crAccount = CentralRepository.getInstance().getAccount(crAccountType, account.getTypeSpecificID()); + + if (crAccount != null && uniqueAccountsList.contains(crAccount) == false) { + uniqueAccountsList.add(crAccount); + } + } + Collection personaAccounts = PersonaAccount.getPersonaAccountsForAccount(account); if (personaAccounts != null && !personaAccounts.isEmpty()) { - - // look for unique accounts - Collection accountCandidates - = personaAccounts - .stream() - .map(PersonaAccount::getAccount) - .collect(Collectors.toList()); - for (CentralRepoAccount crAccount : accountCandidates) { - if (uniqueAccountsList.contains(crAccount) == false) { - uniqueAccountsList.add(crAccount); - } - } - // get personas for the account Collection personas = personaAccounts From 49e844dd6de36e90da9051ae900b5588a973334c Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Thu, 16 Jul 2020 14:42:06 -0400 Subject: [PATCH 022/102] Switch the CVT SummaryViewer file reference container to JList --- .../relationships/Bundle.properties | 3 +- .../relationships/Bundle.properties-MERGED | 4 +- .../relationships/Bundle_ja.properties | 1 - .../relationships/SummaryViewer.form | 98 +++++++++-- .../relationships/SummaryViewer.java | 164 ++++++++++++------ 5 files changed, 199 insertions(+), 71 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties b/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties index b14d8a2688..f6b99d656d 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties @@ -9,7 +9,6 @@ SummaryViewer.callLogsLabel.text=Call Logs: ThreadRootMessagePanel.showAllCheckBox.text=Show All Messages ThreadPane.backButton.text=<--- SummaryViewer.caseReferencesPanel.border.title=Other Occurrences -SummaryViewer.fileReferencesPanel.border.title=File References in Current Case MessageViewer.threadsLabel.text=Select a Thread to View MessageViewer.threadNameLabel.text= MessageViewer.showingMessagesLabel.text=Showing Messages for Thread: @@ -27,3 +26,5 @@ SummaryViewer.referencesLabel.text=Communication References: SummaryViewer.referencesDataLabel.text= SummaryViewer.contactsLabel.text=Book Entries: SummaryViewer.accountCountry.text= +SummaryViewer.fileRefPane.border.title=File Referernce(s) in Current Case +SummaryViewer.selectAccountFileRefLabel.text= SummaryViewer_FileRefNameColumn_Title=Path SummaryViewer_TabTitle=Summary ThreadRootMessagePanel.showAllCheckBox.text=Show All Messages ThreadPane.backButton.text=<--- SummaryViewer.caseReferencesPanel.border.title=Other Occurrences -SummaryViewer.fileReferencesPanel.border.title=File References in Current Case MessageViewer.threadsLabel.text=Select a Thread to View MessageViewer.threadNameLabel.text= MessageViewer.showingMessagesLabel.text=Showing Messages for Thread: @@ -73,3 +73,5 @@ SummaryViewer.referencesLabel.text=Communication References: SummaryViewer.referencesDataLabel.text= SummaryViewer.contactsLabel.text=Book Entries: SummaryViewer.accountCountry.text= +SummaryViewer.fileRefPane.border.title=File Referernce(s) in Current Case +SummaryViewer.selectAccountFileRefLabel.text= -l ` to update properties files based on the newly generated csv file. The csv file should be formatted such that the columns are bundle relative path, property files key, translated value and commit id for the latest commit id for which these changes represent. The commit id only needs to be in the header row. +3. Call `python3 updatepropsscript.py -l ` to update properties files based on the newly generated csv file. The csv file should be formatted such that the columns are bundle relative path, property files key, translated value and commit id for the latest commit id for which these changes represent. The commit id only needs to be in the header row. The output path should be specified as a relative path with the dot slash notation (i.e. `./outputpath.csv`) or an absolute path. ## Localization Generation for the First Time -First-time updates should follow a similar procedure except that instead of calling `diffscript.py`, call `python3 allbundlesscript ` to generate a csv file with relative paths of bundle files, property file keys, property file values. +First-time updates should follow a similar procedure except that instead of calling `diffscript.py`, call `python3 allbundlesscript ` to generate a csv file with relative paths of bundle files, property file keys, property file values. The output path should be specified as a relative path with the dot slash notation (i.e. `./inputpath.csv`) or an absolute path. ##Unit Tests Unit tests can be run from this directory using `python3 -m unittest`. \ No newline at end of file diff --git a/release_scripts/localization_scripts/allbundlesscript.py b/release_scripts/localization_scripts/allbundlesscript.py index 38426d824a..139dec15a5 100644 --- a/release_scripts/localization_scripts/allbundlesscript.py +++ b/release_scripts/localization_scripts/allbundlesscript.py @@ -51,7 +51,9 @@ def main(): parser = argparse.ArgumentParser(description='Gathers all key-value pairs within .properties-MERGED files into ' 'one csv file.', formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument(dest='output_path', type=str, help='The path to the output csv file.') + parser.add_argument(dest='output_path', type=str, help='The path to the output csv file. The output path should be' + ' specified as a relative path with the dot slash notation ' + '(i.e. \'./outputpath.csv\') or an absolute path.') parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, help='The path to the repo. If not specified, path of script is used.') parser.add_argument('-nc', '--no_commit', dest='no_commit', action='store_true', default=False, diff --git a/release_scripts/localization_scripts/diffscript.py b/release_scripts/localization_scripts/diffscript.py index 259191292e..2713fef518 100644 --- a/release_scripts/localization_scripts/diffscript.py +++ b/release_scripts/localization_scripts/diffscript.py @@ -54,7 +54,9 @@ def main(): "'.properties-MERGED' files and generates a csv file containing " "the items changed.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument(dest='output_path', type=str, help='The path to the output csv file.') + parser.add_argument(dest='output_path', type=str, help='The path to the output csv file. The output path should ' + 'be specified as a relative path with the dot slash notation' + ' (i.e. \'./outputpath.csv\') or an absolute path.') parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, help='The path to the repo. If not specified, path of script is used.') diff --git a/release_scripts/localization_scripts/updatepropsscript.py b/release_scripts/localization_scripts/updatepropsscript.py index 2663ee330a..3d8489af82 100644 --- a/release_scripts/localization_scripts/updatepropsscript.py +++ b/release_scripts/localization_scripts/updatepropsscript.py @@ -178,7 +178,10 @@ def main(): 'deleted, and commit id for how recent these updates are. ' 'If the key should be deleted, the deletion row should be ' '\'DELETION.\' A header row is expected by default and the ' - 'commit id, if specified, should only be in the first row.') + 'commit id, if specified, should only be in the first row. The' + ' input path should be specified as a relative path with the ' + 'dot slash notation (i.e. `./inputpath.csv`) or an absolute ' + 'path.') parser.add_argument('-r', '--repo', dest='repo_path', type=str, required=False, help='The path to the repo. If not specified, parent repo of path of script is used.')