From 3778b626290c8a9bf8dc06fb8ada8751a32a0a8c Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 16 Mar 2021 14:19:06 -0400 Subject: [PATCH] fixes from comments --- .../autopsy/coreutils/NetworkUtils.java | 45 ++++++--- .../AddEditCategoryDialog.java | 5 +- .../Bundle.properties-MERGED | 8 +- .../WebCategoriesDataModel.java | 94 ++++++++++--------- .../WebCategoriesOptionsPanel.java | 51 +++++++++- 5 files changed, 140 insertions(+), 63 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/NetworkUtils.java b/Core/src/org/sleuthkit/autopsy/coreutils/NetworkUtils.java index 8078a8c6a3..e95e95226e 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/NetworkUtils.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/NetworkUtils.java @@ -55,12 +55,16 @@ public class NetworkUtils { } /** - * Attempt to manually extract the domain from a URL. + * Return the host in the url or empty string if no host can be determined. * - * @param url - * @return empty string if no domain could be found + * @param url The original url-like item. + * @return The host or empty string if no host can be determined. */ - private static String getBaseDomain(String url) { + public static String extractHost(String url) { + if (url == null) { + return ""; + } + String host = null; //strip protocol @@ -74,24 +78,37 @@ public class NetworkUtils { host = cleanUrl; } - String base = host; - try { - base = DomainTokenizer.getInstance().getDomain(host); - } catch (IOException ex) { - logger.log(Level.WARNING, "Unable to load resources for domain categorization.", ex); - } - // verify there are no special characters in there - if (base.matches(".*[~`!@#$%^&\\*\\(\\)\\+={}\\[\\];:\\?<>,/ ].*")) { + if (host.matches(".*[~`!@#$%^&\\*\\(\\)\\+={}\\[\\];:\\?<>,/ ].*")) { return ""; } //verify that the base domain actually has a '.', details JIRA-4609 - if (!base.contains(".")) { + if (!host.contains(".")) { return ""; } - return base; + return host; + } + + /** + * Attempt to manually extract the domain from a URL. + * + * @param url + * @return empty string if no domain could be found + */ + private static String getBaseDomain(String url) { + String base = extractHost(url); + if (StringUtils.isBlank(base)) { + return ""; + } + + try { + return DomainTokenizer.getInstance().getDomain(base); + } catch (IOException ex) { + logger.log(Level.WARNING, "Unable to load resources for domain categorization.", ex); + return ""; + } } /** diff --git a/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/AddEditCategoryDialog.java b/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/AddEditCategoryDialog.java index e4251b6de7..4a8634875a 100644 --- a/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/AddEditCategoryDialog.java +++ b/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/AddEditCategoryDialog.java @@ -127,7 +127,7 @@ class AddEditCategoryDialog extends javax.swing.JDialog { */ @Messages({ "# {0} - maxSuffixLen", - "AddEditCategoryDialog_onValueUpdate_badSuffix=Please provide a domain suffix that is no more than {0} characters that includes at least one period.", + "AddEditCategoryDialog_onValueUpdate_badSuffix=Please provide a valid domain suffix that is no more than {0} characters that includes at least one period.", "# {0} - maxCategoryLen", "AddEditCategoryDialog_onValueUpdate_badCategory=Please provide a category that is no more than {0} characters.", "AddEditCategoryDialog_onValueUpdate_suffixRepeat=Please provide a unique domain suffix.", @@ -150,8 +150,7 @@ class AddEditCategoryDialog extends javax.swing.JDialog { String validationMessage = null; if (normalizedSuffix.length() == 0 - || normalizedSuffix.length() > WebCategoriesDataModel.getMaxDomainSuffixLength() - || safeSuffixStr.indexOf('.') < 0) { + || normalizedSuffix.length() > WebCategoriesDataModel.getMaxDomainSuffixLength()) { validationMessage = Bundle.AddEditCategoryDialog_onValueUpdate_badSuffix(WebCategoriesDataModel.getMaxCategoryLength()); } else if (normalizedCategory.length() == 0 || normalizedCategory.length() > WebCategoriesDataModel.getMaxCategoryLength()) { diff --git a/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/Bundle.properties-MERGED index 78b832f7d5..79ad3708ec 100644 --- a/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/Bundle.properties-MERGED @@ -3,7 +3,7 @@ AddEditCategoryDialog_Edit=Edit Entry # {0} - maxCategoryLen AddEditCategoryDialog_onValueUpdate_badCategory=Please provide a category that is no more than {0} characters. # {0} - maxSuffixLen -AddEditCategoryDialog_onValueUpdate_badSuffix=Please provide a domain suffix that is no more than {0} characters that includes at least one period. +AddEditCategoryDialog_onValueUpdate_badSuffix=Please provide a valid domain suffix that is no more than {0} characters that includes at least one period. AddEditCategoryDialog_onValueUpdate_sameCategory=Please provide a new category for this domain suffix. AddEditCategoryDialog_onValueUpdate_suffixRepeat=Please provide a unique domain suffix. WebCategoriesOptionsPanel_categoryTable_categoryColumnName=Category @@ -15,6 +15,12 @@ WebCategoriesOptionsPanel_exportSetButtonActionPerformed_errorMessage=There was WebCategoriesOptionsPanel_exportSetButtonActionPerformed_errorTitle=Export Error WebCategoriesOptionsPanel_importSetButtonActionPerformed_errorMessage=There was an error importing this json file. WebCategoriesOptionsPanel_importSetButtonActionPerformed_errorTitle=Import Error +WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictCancel=Cancel +# {0} - domainSuffix +WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictMessage=Domain suffix: {0} already exists. What would you like to do? +WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictOverwrite=Overwrite +WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictSkip=Skip +WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictTitle=Domain Suffix Already Exists WebCategoryOptionsController_title=Custom Web Categories WebCategoryOptionsController_keywords=Custom Web Categories AddEditCategoryDialog.categoryLabel.text=Category: diff --git a/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesDataModel.java b/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesDataModel.java index f702063758..ddb622b965 100644 --- a/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesDataModel.java +++ b/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesDataModel.java @@ -41,9 +41,11 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.openide.modules.InstalledFileLocator; +import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.autopsy.url.analytics.DomainCategory; /** @@ -155,7 +157,7 @@ class WebCategoriesDataModel implements AutoCloseable { if (category == null) { return ""; } - + String trimmedCategory = category.trim(); return trimmedCategory.substring(0, Math.min(trimmedCategory.length(), MAX_CAT_SIZE)); @@ -174,8 +176,9 @@ class WebCategoriesDataModel implements AutoCloseable { } String trimmedSuffix = domainSuffix.trim(); - - return trimmedSuffix.substring(0, Math.min(trimmedSuffix.length(), MAX_DOMAIN_SIZE)).toLowerCase(); + String extractedSuffix = NetworkUtils.extractHost(trimmedSuffix); + + return extractedSuffix.substring(0, Math.min(extractedSuffix.length(), MAX_DOMAIN_SIZE)).toLowerCase(); } /** @@ -250,56 +253,32 @@ class WebCategoriesDataModel implements AutoCloseable { } /** - * Imports json file replacing any data in this database. + * Retrieves all domain categories present in json file. * - * @param jsonInput The json file to import. + * @param jsonInput The json file. + * @return The domain categories. * @throws IOException - * @throws SQLException */ - synchronized void importJson(File jsonInput) throws IOException, SQLException { + List getJsonEntries(File jsonInput) throws IOException { if (jsonInput == null) { logger.log(Level.WARNING, "No valid file provided."); - return; - } - - if (!isInitialized()) { - initialize(); + return Collections.emptyList(); } ObjectMapper mapper = new ObjectMapper(); List customCategorizations = mapper.readValue(jsonInput, new TypeReference>() { }); - customCategorizations = customCategorizations == null ? Collections.emptyList() : customCategorizations; + Stream categoryStream = (customCategorizations != null) ? customCategorizations.stream() : Stream.empty(); - // insert all records as a batch for speed purposes - try (PreparedStatement domainInsert = dbConn.prepareStatement( - "INSERT OR REPLACE INTO " + TABLE_NAME + "(" + SUFFIX_COLUMN + ", " + CATEGORY_COLUMN + ") VALUES (?, ?)", Statement.NO_GENERATED_KEYS)) { + return categoryStream + .filter(c -> c != null && c.getCategory() != null && c.getDomains() != null) + .flatMap(c -> c.getDomains().stream() + .map(WebCategoriesDataModel::getNormalizedSuffix) + .filter(StringUtils::isNotBlank) + .map(d -> new DomainCategory(d, getNormalizedCategory(c.getCategory())))) + .collect(Collectors.toList()); - for (int i = 0; i < customCategorizations.size(); i++) { - CustomCategorizationJsonDto category = customCategorizations.get(i); - if (category == null || category.getDomains() == null || category.getCategory() == null) { - logger.log(Level.WARNING, String.format("Could not process item in file: %s at index: %d", jsonInput.getAbsolutePath(), i)); - continue; - } - - String categoryStr = getNormalizedCategory(category.getCategory()); - - for (int listIdx = 0; listIdx < category.getDomains().size(); listIdx++) { - String domain = category.getDomains().get(listIdx); - if (domain == null) { - logger.log(Level.WARNING, String.format("Could not process domain at idx: %d in category %s for file %s", - listIdx, categoryStr, jsonInput.getAbsolutePath())); - } - - domainInsert.setString(1, getNormalizedSuffix(domain)); - domainInsert.setString(2, categoryStr); - domainInsert.addBatch(); - } - } - - domainInsert.executeBatch(); - } } /** @@ -385,8 +364,8 @@ class WebCategoriesDataModel implements AutoCloseable { * @throws IllegalArgumentException */ synchronized boolean insertUpdateSuffix(DomainCategory entry) throws SQLException, IllegalStateException, IllegalArgumentException { - if (entry == null || StringUtils.isBlank(entry.getCategory()) || StringUtils.isBlank(entry.getHostSuffix())) { - throw new IllegalArgumentException("Expected non-empty category and domain suffix."); + if (entry == null || StringUtils.isBlank(getNormalizedCategory(entry.getCategory())) || StringUtils.isBlank(getNormalizedSuffix(entry.getHostSuffix()))) { + throw new IllegalArgumentException("Expected non-empty, valid category and domain suffix."); } if (!isInitialized()) { @@ -430,6 +409,37 @@ class WebCategoriesDataModel implements AutoCloseable { } + private static final String GET_DOMAIN_SUFFIX_QUERY + = "SELECT " + SUFFIX_COLUMN + ", " + CATEGORY_COLUMN + + " FROM " + TABLE_NAME + " WHERE " + SUFFIX_COLUMN + " = ?"; + + /** + * Return the matching domain suffix or null if none found. + * + * @param domainSuffix The domain suffix. + * @return The found entry or null. + * @throws SQLException + */ + DomainCategory getRecordBySuffix(String domainSuffix) throws SQLException { + if (!isInitialized()) { + initialize(); + } + + try (PreparedStatement domainSelect = dbConn.prepareStatement(GET_DOMAIN_SUFFIX_QUERY)) { + domainSelect.setString(1, domainSuffix); + + try (ResultSet resultSet = domainSelect.executeQuery()) { + if (resultSet.next()) { + return new DomainCategory( + resultSet.getString(SUFFIX_COLUMN), + resultSet.getString(CATEGORY_COLUMN)); + } else { + return null; + } + } + } + } + // get the suffix and category from the main table and gets the longest matching suffix. private static final String BASE_QUERY_FMT_STR = "SELECT " + SUFFIX_COLUMN + ", " + CATEGORY_COLUMN + " FROM " + TABLE_NAME diff --git a/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesOptionsPanel.java b/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesOptionsPanel.java index cd39a8d388..d816764e26 100644 --- a/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesOptionsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesOptionsPanel.java @@ -36,6 +36,7 @@ import javax.swing.JOptionPane; import javax.swing.SwingUtilities; import javax.swing.filechooser.FileNameExtensionFilter; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.openide.util.NbBundle.Messages; import org.openide.util.WeakListeners; import org.sleuthkit.autopsy.corecomponents.OptionsPanel; @@ -76,7 +77,7 @@ public class WebCategoriesOptionsPanel extends IngestModuleGlobalSettingsPanel i new ColumnModel<>( Bundle.WebCategoriesOptionsPanel_categoryTable_categoryColumnName(), (domCat) -> new DefaultCellModel<>(domCat.getCategory()) - .setTooltip(domCat.getCategory()), + .setTooltip(domCat.getCategory()), 200 ) )).setKeyFunction((domCat) -> domCat.getHostSuffix()); @@ -433,7 +434,13 @@ public class WebCategoriesOptionsPanel extends IngestModuleGlobalSettingsPanel i @Messages({ "WebCategoriesOptionsPanel_importSetButtonActionPerformed_errorMessage=There was an error importing this json file.", - "WebCategoriesOptionsPanel_importSetButtonActionPerformed_errorTitle=Import Error",}) + "WebCategoriesOptionsPanel_importSetButtonActionPerformed_errorTitle=Import Error", + "WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictTitle=Domain Suffix Already Exists", + "# {0} - domainSuffix", + "WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictMessage=Domain suffix: {0} already exists. What would you like to do?", + "WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictOverwrite=Overwrite", + "WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictSkip=Skip", + "WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictCancel=Cancel"}) private void importSetButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_importSetButtonActionPerformed fileChooser.setSelectedFile(new File("")); int result = fileChooser.showOpenDialog(this); @@ -441,7 +448,45 @@ public class WebCategoriesOptionsPanel extends IngestModuleGlobalSettingsPanel i File selectedFile = fileChooser.getSelectedFile(); if (selectedFile != null && selectedFile.exists()) { try { - runUpdateAction(() -> dataModel.importJson(selectedFile)); + runUpdateAction(() -> { + List categories = dataModel.getJsonEntries(selectedFile); + + for (DomainCategory domcat : categories) { + String normalizedCategory = domcat == null ? "" : WebCategoriesDataModel.getNormalizedCategory(domcat.getCategory()); + String normalizedSuffix = domcat == null ? "" : WebCategoriesDataModel.getNormalizedSuffix(domcat.getHostSuffix()); + + if (StringUtils.isBlank(normalizedCategory) || StringUtils.isBlank(normalizedSuffix)) { + logger.log(Level.WARNING, String.format("Invalid entry [category: %s, domain suffix: %s]", normalizedCategory, normalizedSuffix)); + continue; + } + + DomainCategory currentCategory = dataModel.getRecordBySuffix(normalizedSuffix); + // if a mapping for the domain suffix already exists and the value will change, prompt the user on what to do. + if (currentCategory != null && !normalizedCategory.equalsIgnoreCase(currentCategory.getCategory())) { + String[] options = { + Bundle.WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictOverwrite(), + Bundle.WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictSkip(), + Bundle.WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictCancel() + }; + + int optionItem = JOptionPane.showOptionDialog(null, + Bundle.WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictMessage(normalizedSuffix), + Bundle.WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictTitle(), + JOptionPane.DEFAULT_OPTION, JOptionPane.WARNING_MESSAGE, null, options, options[0]); + + switch (optionItem) { + case 0: + break; + case 1: + continue; + case 2: + return; + } + } + + dataModel.insertUpdateSuffix(new DomainCategory(normalizedSuffix, normalizedCategory)); + } + }); } catch (IllegalArgumentException | SQLException | IOException ex) { setDefaultCursor(); JOptionPane.showMessageDialog(