fixes from comments

Core/src/org/sleuthkit/autopsy/coreutils/NetworkUtils.java

@@ -55,12 +55,16 @@ public class NetworkUtils {
     }
 
     /**
-     * Attempt to manually extract the domain from a URL.
+     * Return the host in the url or empty string if no host can be determined.
      *
-     * @param url
+     * @param url The original url-like item.
-     * @return empty string if no domain could be found
+     * @return The host or empty string if no host can be determined.
      */
-    private static String getBaseDomain(String url) {
+    public static String extractHost(String url) {
+        if (url == null) {
+            return "";
+        }
+
         String host = null;
 
         //strip protocol
@@ -74,24 +78,37 @@ public class NetworkUtils {
             host = cleanUrl;
         }
 
-        String base = host;
-        try {
-            base = DomainTokenizer.getInstance().getDomain(host);
-        } catch (IOException ex) {
-            logger.log(Level.WARNING, "Unable to load resources for domain categorization.", ex);
-        }
-
         // verify there are no special characters in there
-        if (base.matches(".*[~`!@#$%^&\\*\\(\\)\\+={}\\[\\];:\\?<>,/ ].*")) {
+        if (host.matches(".*[~`!@#$%^&\\*\\(\\)\\+={}\\[\\];:\\?<>,/ ].*")) {
             return "";
         }
 
         //verify that the base domain actually has a '.', details JIRA-4609
-        if (!base.contains(".")) {
+        if (!host.contains(".")) {
             return "";
         }
 
-        return base;
+        return host;
+    }
+
+    /**
+     * Attempt to manually extract the domain from a URL.
+     *
+     * @param url
+     * @return empty string if no domain could be found
+     */
+    private static String getBaseDomain(String url) {
+        String base = extractHost(url);
+        if (StringUtils.isBlank(base)) {
+            return "";
+        }
+
+        try {
+            return DomainTokenizer.getInstance().getDomain(base);
+        } catch (IOException ex) {
+            logger.log(Level.WARNING, "Unable to load resources for domain categorization.", ex);
+            return "";
+        }
     }
 
     /**

Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/AddEditCategoryDialog.java

@@ -127,7 +127,7 @@ class AddEditCategoryDialog extends javax.swing.JDialog {
      */
     @Messages({
         "# {0} - maxSuffixLen",
-        "AddEditCategoryDialog_onValueUpdate_badSuffix=Please provide a domain suffix that is no more than {0} characters that includes at least one period.",
+        "AddEditCategoryDialog_onValueUpdate_badSuffix=Please provide a valid domain suffix that is no more than {0} characters that includes at least one period.",
         "# {0} - maxCategoryLen",
         "AddEditCategoryDialog_onValueUpdate_badCategory=Please provide a category that is no more than {0} characters.",
         "AddEditCategoryDialog_onValueUpdate_suffixRepeat=Please provide a unique domain suffix.",
@@ -150,8 +150,7 @@ class AddEditCategoryDialog extends javax.swing.JDialog {
 
         String validationMessage = null;
         if (normalizedSuffix.length() == 0
-                || normalizedSuffix.length() > WebCategoriesDataModel.getMaxDomainSuffixLength()
+                || normalizedSuffix.length() > WebCategoriesDataModel.getMaxDomainSuffixLength()) {
-                || safeSuffixStr.indexOf('.') < 0) {
             validationMessage = Bundle.AddEditCategoryDialog_onValueUpdate_badSuffix(WebCategoriesDataModel.getMaxCategoryLength());
 
         } else if (normalizedCategory.length() == 0 || normalizedCategory.length() > WebCategoriesDataModel.getMaxCategoryLength()) {

Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/Bundle.properties-MERGED

@@ -3,7 +3,7 @@ AddEditCategoryDialog_Edit=Edit Entry
 # {0} - maxCategoryLen
 AddEditCategoryDialog_onValueUpdate_badCategory=Please provide a category that is no more than {0} characters.
 # {0} - maxSuffixLen
-AddEditCategoryDialog_onValueUpdate_badSuffix=Please provide a domain suffix that is no more than {0} characters that includes at least one period.
+AddEditCategoryDialog_onValueUpdate_badSuffix=Please provide a valid domain suffix that is no more than {0} characters that includes at least one period.
 AddEditCategoryDialog_onValueUpdate_sameCategory=Please provide a new category for this domain suffix.
 AddEditCategoryDialog_onValueUpdate_suffixRepeat=Please provide a unique domain suffix.
 WebCategoriesOptionsPanel_categoryTable_categoryColumnName=Category
@@ -15,6 +15,12 @@ WebCategoriesOptionsPanel_exportSetButtonActionPerformed_errorMessage=There was
 WebCategoriesOptionsPanel_exportSetButtonActionPerformed_errorTitle=Export Error
 WebCategoriesOptionsPanel_importSetButtonActionPerformed_errorMessage=There was an error importing this json file.
 WebCategoriesOptionsPanel_importSetButtonActionPerformed_errorTitle=Import Error
+WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictCancel=Cancel
+# {0} - domainSuffix
+WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictMessage=Domain suffix: {0} already exists. What would you like to do?
+WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictOverwrite=Overwrite
+WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictSkip=Skip
+WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictTitle=Domain Suffix Already Exists
 WebCategoryOptionsController_title=Custom Web Categories
 WebCategoryOptionsController_keywords=Custom Web Categories
 AddEditCategoryDialog.categoryLabel.text=Category:

Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesDataModel.java

@@ -41,9 +41,11 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
+import java.util.stream.Stream;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.openide.modules.InstalledFileLocator;
+import org.sleuthkit.autopsy.coreutils.NetworkUtils;
 import org.sleuthkit.autopsy.url.analytics.DomainCategory;
 
 /**
@@ -174,8 +176,9 @@ class WebCategoriesDataModel implements AutoCloseable {
         }
 
         String trimmedSuffix = domainSuffix.trim();
+        String extractedSuffix = NetworkUtils.extractHost(trimmedSuffix);
 
-        return trimmedSuffix.substring(0, Math.min(trimmedSuffix.length(), MAX_DOMAIN_SIZE)).toLowerCase();
+        return extractedSuffix.substring(0, Math.min(extractedSuffix.length(), MAX_DOMAIN_SIZE)).toLowerCase();
     }
 
     /**
@@ -250,56 +253,32 @@ class WebCategoriesDataModel implements AutoCloseable {
     }
 
     /**
-     * Imports json file replacing any data in this database.
+     * Retrieves all domain categories present in json file.
      *
-     * @param jsonInput The json file to import.
+     * @param jsonInput The json file.
+     * @return The domain categories.
      * @throws IOException
-     * @throws SQLException
      */
-    synchronized void importJson(File jsonInput) throws IOException, SQLException {
+    List<DomainCategory> getJsonEntries(File jsonInput) throws IOException {
         if (jsonInput == null) {
             logger.log(Level.WARNING, "No valid file provided.");
-            return;
+            return Collections.emptyList();
-        }
-
-        if (!isInitialized()) {
-            initialize();
         }
 
         ObjectMapper mapper = new ObjectMapper();
         List<CustomCategorizationJsonDto> customCategorizations = mapper.readValue(jsonInput, new TypeReference<List<CustomCategorizationJsonDto>>() {
         });
 
-        customCategorizations = customCategorizations == null ? Collections.emptyList() : customCategorizations;
+        Stream<CustomCategorizationJsonDto> categoryStream = (customCategorizations != null) ? customCategorizations.stream() : Stream.empty();
 
-        // insert all records as a batch for speed purposes
+        return categoryStream
-        try (PreparedStatement domainInsert = dbConn.prepareStatement(
+                .filter(c -> c != null && c.getCategory() != null && c.getDomains() != null)
-                "INSERT OR REPLACE INTO " + TABLE_NAME + "(" + SUFFIX_COLUMN + ", " + CATEGORY_COLUMN + ") VALUES (?, ?)", Statement.NO_GENERATED_KEYS)) {
+                .flatMap(c -> c.getDomains().stream()
+                .map(WebCategoriesDataModel::getNormalizedSuffix)
+                .filter(StringUtils::isNotBlank)
+                .map(d -> new DomainCategory(d, getNormalizedCategory(c.getCategory()))))
+                .collect(Collectors.toList());
 
-            for (int i = 0; i < customCategorizations.size(); i++) {
-                CustomCategorizationJsonDto category = customCategorizations.get(i);
-                if (category == null || category.getDomains() == null || category.getCategory() == null) {
-                    logger.log(Level.WARNING, String.format("Could not process item in file: %s at index: %d", jsonInput.getAbsolutePath(), i));
-                    continue;
-                }
-
-                String categoryStr = getNormalizedCategory(category.getCategory());
-
-                for (int listIdx = 0; listIdx < category.getDomains().size(); listIdx++) {
-                    String domain = category.getDomains().get(listIdx);
-                    if (domain == null) {
-                        logger.log(Level.WARNING, String.format("Could not process domain at idx: %d in category %s for file %s",
-                                listIdx, categoryStr, jsonInput.getAbsolutePath()));
-                    }
-
-                    domainInsert.setString(1, getNormalizedSuffix(domain));
-                    domainInsert.setString(2, categoryStr);
-                    domainInsert.addBatch();
-                }
-            }
-
-            domainInsert.executeBatch();
-        }
     }
 
     /**
@@ -385,8 +364,8 @@ class WebCategoriesDataModel implements AutoCloseable {
      * @throws IllegalArgumentException
      */
     synchronized boolean insertUpdateSuffix(DomainCategory entry) throws SQLException, IllegalStateException, IllegalArgumentException {
-        if (entry == null || StringUtils.isBlank(entry.getCategory()) || StringUtils.isBlank(entry.getHostSuffix())) {
+        if (entry == null || StringUtils.isBlank(getNormalizedCategory(entry.getCategory())) || StringUtils.isBlank(getNormalizedSuffix(entry.getHostSuffix()))) {
-            throw new IllegalArgumentException("Expected non-empty category and domain suffix.");
+            throw new IllegalArgumentException("Expected non-empty, valid category and domain suffix.");
         }
 
         if (!isInitialized()) {
@@ -430,6 +409,37 @@ class WebCategoriesDataModel implements AutoCloseable {
 
     }
 
+    private static final String GET_DOMAIN_SUFFIX_QUERY
+            = "SELECT " + SUFFIX_COLUMN + ", " + CATEGORY_COLUMN
+            + " FROM " + TABLE_NAME + " WHERE " + SUFFIX_COLUMN + " = ?";
+
+    /**
+     * Return the matching domain suffix or null if none found.
+     *
+     * @param domainSuffix The domain suffix.
+     * @return The found entry or null.
+     * @throws SQLException
+     */
+    DomainCategory getRecordBySuffix(String domainSuffix) throws SQLException {
+        if (!isInitialized()) {
+            initialize();
+        }
+
+        try (PreparedStatement domainSelect = dbConn.prepareStatement(GET_DOMAIN_SUFFIX_QUERY)) {
+            domainSelect.setString(1, domainSuffix);
+
+            try (ResultSet resultSet = domainSelect.executeQuery()) {
+                if (resultSet.next()) {
+                    return new DomainCategory(
+                            resultSet.getString(SUFFIX_COLUMN),
+                            resultSet.getString(CATEGORY_COLUMN));
+                } else {
+                    return null;
+                }
+            }
+        }
+    }
+
     // get the suffix and category from the main table and gets the longest matching suffix.
     private static final String BASE_QUERY_FMT_STR
             = "SELECT " + SUFFIX_COLUMN + ", " + CATEGORY_COLUMN + " FROM " + TABLE_NAME

Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesOptionsPanel.java

@@ -36,6 +36,7 @@ import javax.swing.JOptionPane;
 import javax.swing.SwingUtilities;
 import javax.swing.filechooser.FileNameExtensionFilter;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
 import org.openide.util.NbBundle.Messages;
 import org.openide.util.WeakListeners;
 import org.sleuthkit.autopsy.corecomponents.OptionsPanel;
@@ -76,7 +77,7 @@ public class WebCategoriesOptionsPanel extends IngestModuleGlobalSettingsPanel i
                     new ColumnModel<>(
                             Bundle.WebCategoriesOptionsPanel_categoryTable_categoryColumnName(),
                             (domCat) -> new DefaultCellModel<>(domCat.getCategory())
-                                .setTooltip(domCat.getCategory()),
+                                    .setTooltip(domCat.getCategory()),
                             200
                     )
             )).setKeyFunction((domCat) -> domCat.getHostSuffix());
@@ -433,7 +434,13 @@ public class WebCategoriesOptionsPanel extends IngestModuleGlobalSettingsPanel i
 
     @Messages({
         "WebCategoriesOptionsPanel_importSetButtonActionPerformed_errorMessage=There was an error importing this json file.",
-        "WebCategoriesOptionsPanel_importSetButtonActionPerformed_errorTitle=Import Error",})
+        "WebCategoriesOptionsPanel_importSetButtonActionPerformed_errorTitle=Import Error",
+        "WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictTitle=Domain Suffix Already Exists",
+        "# {0} - domainSuffix",
+        "WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictMessage=Domain suffix: {0} already exists. What would you like to do?",
+        "WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictOverwrite=Overwrite",
+        "WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictSkip=Skip",
+        "WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictCancel=Cancel"})
     private void importSetButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_importSetButtonActionPerformed
         fileChooser.setSelectedFile(new File(""));
         int result = fileChooser.showOpenDialog(this);
@@ -441,7 +448,45 @@ public class WebCategoriesOptionsPanel extends IngestModuleGlobalSettingsPanel i
             File selectedFile = fileChooser.getSelectedFile();
             if (selectedFile != null && selectedFile.exists()) {
                 try {
-                    runUpdateAction(() -> dataModel.importJson(selectedFile));
+                    runUpdateAction(() -> {
+                        List<DomainCategory> categories = dataModel.getJsonEntries(selectedFile);
+
+                        for (DomainCategory domcat : categories) {
+                            String normalizedCategory = domcat == null ? "" : WebCategoriesDataModel.getNormalizedCategory(domcat.getCategory());
+                            String normalizedSuffix = domcat == null ? "" : WebCategoriesDataModel.getNormalizedSuffix(domcat.getHostSuffix());
+
+                            if (StringUtils.isBlank(normalizedCategory) || StringUtils.isBlank(normalizedSuffix)) {
+                                logger.log(Level.WARNING, String.format("Invalid entry [category: %s, domain suffix: %s]", normalizedCategory, normalizedSuffix));
+                                continue;
+                            }
+
+                            DomainCategory currentCategory = dataModel.getRecordBySuffix(normalizedSuffix);
+                            // if a mapping for the domain suffix already exists and the value will change, prompt the user on what to do.
+                            if (currentCategory != null && !normalizedCategory.equalsIgnoreCase(currentCategory.getCategory())) {
+                                String[] options = {
+                                    Bundle.WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictOverwrite(),
+                                    Bundle.WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictSkip(),
+                                    Bundle.WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictCancel()
+                                };
+
+                                int optionItem = JOptionPane.showOptionDialog(null,
+                                        Bundle.WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictMessage(normalizedSuffix),
+                                        Bundle.WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictTitle(),
+                                        JOptionPane.DEFAULT_OPTION, JOptionPane.WARNING_MESSAGE, null, options, options[0]);
+
+                                switch (optionItem) {
+                                    case 0:
+                                        break;
+                                    case 1:
+                                        continue;
+                                    case 2:
+                                        return;
+                                }
+                            }
+
+                            dataModel.insertUpdateSuffix(new DomainCategory(normalizedSuffix, normalizedCategory));
+                        }
+                    });
                 } catch (IllegalArgumentException | SQLException | IOException ex) {
                     setDefaultCursor();
                     JOptionPane.showMessageDialog(