mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-17 10:17:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixes from comments

Browse Source
This commit is contained in:
Greg DiCristofaro 2021-03-16 14:19:06 -04:00
parent 8797705bac
commit 3778b62629
5 changed files with 140 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
Core/src/org/sleuthkit/autopsy/coreutils/NetworkUtils.java
View File

@ -55,12 +55,16 @@ public class NetworkUtils {
}
/**
* Attempt to manually extract the domain from a URL.
* Return the host in the url or empty string if no host can be determined.
*
* @param url
* @return empty string if no domain could be found
* @param url The original url-like item.
* @return The host or empty string if no host can be determined.
*/
private static String getBaseDomain(String url) {
public static String extractHost(String url) {
if (url == null) {
return "";
}
String host = null;
//strip protocol
@ -74,24 +78,37 @@ public class NetworkUtils {
host = cleanUrl;
}
String base = host;
try {
base = DomainTokenizer.getInstance().getDomain(host);
} catch (IOException ex) {
logger.log(Level.WARNING, "Unable to load resources for domain categorization.", ex);
}
// verify there are no special characters in there
if (base.matches(".*[~`!@#$%^&\\*\\(\\)\\+={}\\[\\];:\\?<>,/ ].*")) {
if (host.matches(".*[~`!@#$%^&\\*\\(\\)\\+={}\\[\\];:\\?<>,/ ].*")) {
return "";
}
//verify that the base domain actually has a '.', details JIRA-4609
if (!base.contains(".")) {
if (!host.contains(".")) {
return "";
}
return base;
return host;
}
/**
* Attempt to manually extract the domain from a URL.
*
* @param url
* @return empty string if no domain could be found
*/
private static String getBaseDomain(String url) {
String base = extractHost(url);
if (StringUtils.isBlank(base)) {
return "";
}
try {
return DomainTokenizer.getInstance().getDomain(base);
} catch (IOException ex) {
logger.log(Level.WARNING, "Unable to load resources for domain categorization.", ex);
return "";
}
}
/**

5
Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/AddEditCategoryDialog.java
View File

@ -127,7 +127,7 @@ class AddEditCategoryDialog extends javax.swing.JDialog {
*/
@Messages({
"# {0} - maxSuffixLen",
"AddEditCategoryDialog_onValueUpdate_badSuffix=Please provide a domain suffix that is no more than {0} characters that includes at least one period.",
"AddEditCategoryDialog_onValueUpdate_badSuffix=Please provide a valid domain suffix that is no more than {0} characters that includes at least one period.",
"# {0} - maxCategoryLen",
"AddEditCategoryDialog_onValueUpdate_badCategory=Please provide a category that is no more than {0} characters.",
"AddEditCategoryDialog_onValueUpdate_suffixRepeat=Please provide a unique domain suffix.",
@ -150,8 +150,7 @@ class AddEditCategoryDialog extends javax.swing.JDialog {
String validationMessage = null;
if (normalizedSuffix.length() == 0
|| normalizedSuffix.length() > WebCategoriesDataModel.getMaxDomainSuffixLength()
|| safeSuffixStr.indexOf('.') < 0) {
|| normalizedSuffix.length() > WebCategoriesDataModel.getMaxDomainSuffixLength()) {
validationMessage = Bundle.AddEditCategoryDialog_onValueUpdate_badSuffix(WebCategoriesDataModel.getMaxCategoryLength());
} else if (normalizedCategory.length() == 0 || normalizedCategory.length() > WebCategoriesDataModel.getMaxCategoryLength()) {

8
Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/Bundle.properties-MERGED
View File

@ -3,7 +3,7 @@ AddEditCategoryDialog_Edit=Edit Entry
# {0} - maxCategoryLen
AddEditCategoryDialog_onValueUpdate_badCategory=Please provide a category that is no more than {0} characters.
# {0} - maxSuffixLen
AddEditCategoryDialog_onValueUpdate_badSuffix=Please provide a domain suffix that is no more than {0} characters that includes at least one period.
AddEditCategoryDialog_onValueUpdate_badSuffix=Please provide a valid domain suffix that is no more than {0} characters that includes at least one period.
AddEditCategoryDialog_onValueUpdate_sameCategory=Please provide a new category for this domain suffix.
AddEditCategoryDialog_onValueUpdate_suffixRepeat=Please provide a unique domain suffix.
WebCategoriesOptionsPanel_categoryTable_categoryColumnName=Category
@ -15,6 +15,12 @@ WebCategoriesOptionsPanel_exportSetButtonActionPerformed_errorMessage=There was
WebCategoriesOptionsPanel_exportSetButtonActionPerformed_errorTitle=Export Error
WebCategoriesOptionsPanel_importSetButtonActionPerformed_errorMessage=There was an error importing this json file.
WebCategoriesOptionsPanel_importSetButtonActionPerformed_errorTitle=Import Error
WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictCancel=Cancel
# {0} - domainSuffix
WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictMessage=Domain suffix: {0} already exists. What would you like to do?
WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictOverwrite=Overwrite
WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictSkip=Skip
WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictTitle=Domain Suffix Already Exists
WebCategoryOptionsController_title=Custom Web Categories
WebCategoryOptionsController_keywords=Custom Web Categories
AddEditCategoryDialog.categoryLabel.text=Category:

94
Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesDataModel.java
View File

@ -41,9 +41,11 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.openide.modules.InstalledFileLocator;
import org.sleuthkit.autopsy.coreutils.NetworkUtils;
import org.sleuthkit.autopsy.url.analytics.DomainCategory;
/**
@ -155,7 +157,7 @@ class WebCategoriesDataModel implements AutoCloseable {
if (category == null) {
return "";
}
String trimmedCategory = category.trim();
return trimmedCategory.substring(0, Math.min(trimmedCategory.length(), MAX_CAT_SIZE));
@ -174,8 +176,9 @@ class WebCategoriesDataModel implements AutoCloseable {
}
String trimmedSuffix = domainSuffix.trim();
return trimmedSuffix.substring(0, Math.min(trimmedSuffix.length(), MAX_DOMAIN_SIZE)).toLowerCase();
String extractedSuffix = NetworkUtils.extractHost(trimmedSuffix);
return extractedSuffix.substring(0, Math.min(extractedSuffix.length(), MAX_DOMAIN_SIZE)).toLowerCase();
}
/**
@ -250,56 +253,32 @@ class WebCategoriesDataModel implements AutoCloseable {
}
/**
* Imports json file replacing any data in this database.
* Retrieves all domain categories present in json file.
*
* @param jsonInput The json file to import.
* @param jsonInput The json file.
* @return The domain categories.
* @throws IOException
* @throws SQLException
*/
synchronized void importJson(File jsonInput) throws IOException, SQLException {
List<DomainCategory> getJsonEntries(File jsonInput) throws IOException {
if (jsonInput == null) {
logger.log(Level.WARNING, "No valid file provided.");
return;
}
if (!isInitialized()) {
initialize();
return Collections.emptyList();
}
ObjectMapper mapper = new ObjectMapper();
List<CustomCategorizationJsonDto> customCategorizations = mapper.readValue(jsonInput, new TypeReference<List<CustomCategorizationJsonDto>>() {
});
customCategorizations = customCategorizations == null ? Collections.emptyList() : customCategorizations;
Stream<CustomCategorizationJsonDto> categoryStream = (customCategorizations != null) ? customCategorizations.stream() : Stream.empty();
// insert all records as a batch for speed purposes
try (PreparedStatement domainInsert = dbConn.prepareStatement(
"INSERT OR REPLACE INTO " + TABLE_NAME + "(" + SUFFIX_COLUMN + ", " + CATEGORY_COLUMN + ") VALUES (?, ?)", Statement.NO_GENERATED_KEYS)) {
return categoryStream
.filter(c -> c != null && c.getCategory() != null && c.getDomains() != null)
.flatMap(c -> c.getDomains().stream()
.map(WebCategoriesDataModel::getNormalizedSuffix)
.filter(StringUtils::isNotBlank)
.map(d -> new DomainCategory(d, getNormalizedCategory(c.getCategory()))))
.collect(Collectors.toList());
for (int i = 0; i < customCategorizations.size(); i++) {
CustomCategorizationJsonDto category = customCategorizations.get(i);
if (category == null || category.getDomains() == null || category.getCategory() == null) {
logger.log(Level.WARNING, String.format("Could not process item in file: %s at index: %d", jsonInput.getAbsolutePath(), i));
continue;
}
String categoryStr = getNormalizedCategory(category.getCategory());
for (int listIdx = 0; listIdx < category.getDomains().size(); listIdx++) {
String domain = category.getDomains().get(listIdx);
if (domain == null) {
logger.log(Level.WARNING, String.format("Could not process domain at idx: %d in category %s for file %s",
listIdx, categoryStr, jsonInput.getAbsolutePath()));
}
domainInsert.setString(1, getNormalizedSuffix(domain));
domainInsert.setString(2, categoryStr);
domainInsert.addBatch();
}
}
domainInsert.executeBatch();
}
}
/**
@ -385,8 +364,8 @@ class WebCategoriesDataModel implements AutoCloseable {
* @throws IllegalArgumentException
*/
synchronized boolean insertUpdateSuffix(DomainCategory entry) throws SQLException, IllegalStateException, IllegalArgumentException {
if (entry == null || StringUtils.isBlank(entry.getCategory()) || StringUtils.isBlank(entry.getHostSuffix())) {
throw new IllegalArgumentException("Expected non-empty category and domain suffix.");
if (entry == null || StringUtils.isBlank(getNormalizedCategory(entry.getCategory())) || StringUtils.isBlank(getNormalizedSuffix(entry.getHostSuffix()))) {
throw new IllegalArgumentException("Expected non-empty, valid category and domain suffix.");
}
if (!isInitialized()) {
@ -430,6 +409,37 @@ class WebCategoriesDataModel implements AutoCloseable {
}
private static final String GET_DOMAIN_SUFFIX_QUERY
= "SELECT " + SUFFIX_COLUMN + ", " + CATEGORY_COLUMN
+ " FROM " + TABLE_NAME + " WHERE " + SUFFIX_COLUMN + " = ?";
/**
* Return the matching domain suffix or null if none found.
*
* @param domainSuffix The domain suffix.
* @return The found entry or null.
* @throws SQLException
*/
DomainCategory getRecordBySuffix(String domainSuffix) throws SQLException {
if (!isInitialized()) {
initialize();
}
try (PreparedStatement domainSelect = dbConn.prepareStatement(GET_DOMAIN_SUFFIX_QUERY)) {
domainSelect.setString(1, domainSuffix);
try (ResultSet resultSet = domainSelect.executeQuery()) {
if (resultSet.next()) {
return new DomainCategory(
resultSet.getString(SUFFIX_COLUMN),
resultSet.getString(CATEGORY_COLUMN));
} else {
return null;
}
}
}
}
// get the suffix and category from the main table and gets the longest matching suffix.
private static final String BASE_QUERY_FMT_STR
= "SELECT " + SUFFIX_COLUMN + ", " + CATEGORY_COLUMN + " FROM " + TABLE_NAME

51
Core/src/org/sleuthkit/autopsy/url/analytics/domaincategorization/WebCategoriesOptionsPanel.java
View File

@ -36,6 +36,7 @@ import javax.swing.JOptionPane;
import javax.swing.SwingUtilities;
import javax.swing.filechooser.FileNameExtensionFilter;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.openide.util.NbBundle.Messages;
import org.openide.util.WeakListeners;
import org.sleuthkit.autopsy.corecomponents.OptionsPanel;
@ -76,7 +77,7 @@ public class WebCategoriesOptionsPanel extends IngestModuleGlobalSettingsPanel i
new ColumnModel<>(
Bundle.WebCategoriesOptionsPanel_categoryTable_categoryColumnName(),
(domCat) -> new DefaultCellModel<>(domCat.getCategory())
.setTooltip(domCat.getCategory()),
.setTooltip(domCat.getCategory()),
200
)
)).setKeyFunction((domCat) -> domCat.getHostSuffix());
@ -433,7 +434,13 @@ public class WebCategoriesOptionsPanel extends IngestModuleGlobalSettingsPanel i
@Messages({
"WebCategoriesOptionsPanel_importSetButtonActionPerformed_errorMessage=There was an error importing this json file.",
"WebCategoriesOptionsPanel_importSetButtonActionPerformed_errorTitle=Import Error",})
"WebCategoriesOptionsPanel_importSetButtonActionPerformed_errorTitle=Import Error",
"WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictTitle=Domain Suffix Already Exists",
"# {0} - domainSuffix",
"WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictMessage=Domain suffix: {0} already exists. What would you like to do?",
"WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictOverwrite=Overwrite",
"WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictSkip=Skip",
"WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictCancel=Cancel"})
private void importSetButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_importSetButtonActionPerformed
fileChooser.setSelectedFile(new File(""));
int result = fileChooser.showOpenDialog(this);
@ -441,7 +448,45 @@ public class WebCategoriesOptionsPanel extends IngestModuleGlobalSettingsPanel i
File selectedFile = fileChooser.getSelectedFile();
if (selectedFile != null && selectedFile.exists()) {
try {
runUpdateAction(() -> dataModel.importJson(selectedFile));
runUpdateAction(() -> {
List<DomainCategory> categories = dataModel.getJsonEntries(selectedFile);
for (DomainCategory domcat : categories) {
String normalizedCategory = domcat == null ? "" : WebCategoriesDataModel.getNormalizedCategory(domcat.getCategory());
String normalizedSuffix = domcat == null ? "" : WebCategoriesDataModel.getNormalizedSuffix(domcat.getHostSuffix());
if (StringUtils.isBlank(normalizedCategory) || StringUtils.isBlank(normalizedSuffix)) {
logger.log(Level.WARNING, String.format("Invalid entry [category: %s, domain suffix: %s]", normalizedCategory, normalizedSuffix));
continue;
}
DomainCategory currentCategory = dataModel.getRecordBySuffix(normalizedSuffix);
// if a mapping for the domain suffix already exists and the value will change, prompt the user on what to do.
if (currentCategory != null && !normalizedCategory.equalsIgnoreCase(currentCategory.getCategory())) {
String[] options = {
Bundle.WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictOverwrite(),
Bundle.WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictSkip(),
Bundle.WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictCancel()
};
int optionItem = JOptionPane.showOptionDialog(null,
Bundle.WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictMessage(normalizedSuffix),
Bundle.WebCategoriesOptionsPanel_importSetButtonActionPerformed_onConflictTitle(),
JOptionPane.DEFAULT_OPTION, JOptionPane.WARNING_MESSAGE, null, options, options[0]);
switch (optionItem) {
case 0:
break;
case 1:
continue;
case 2:
return;
}
}
dataModel.insertUpdateSuffix(new DomainCategory(normalizedSuffix, normalizedCategory));
}
});
} catch (IllegalArgumentException | SQLException | IOException ex) {
setDefaultCursor();
JOptionPane.showMessageDialog(