diff --git a/.gitignore b/.gitignore index 954207ceec..faa1b92a94 100755 --- a/.gitignore +++ b/.gitignore @@ -1,91 +1,103 @@ -/dist/ -/build/ -/*/build/ -*/nbproject/private/* -/nbproject/private/* - -/Core/release/ -/Core/src/org/sleuthkit/autopsy/coreutils/Version.properties -/Core/src/org/sleuthkit/autopsy/casemodule/docs/QuickStart.html -/Core/src/org/sleuthkit/autopsy/casemodule/docs/screenshot.png -/Core/src/org/sleuthkit/autopsy/datamodel/ranges.csv -/Core/build/ -/Core/dist/ -/Core/nbproject/* -/Core/test/qa-functional/data/* -!/Core/nbproject/project.xml -!/Core/nbproject/project.properties - -/CoreLibs/release/ -/CoreLibs/build/ -/CoreLibs/dist/ -/CoreLibs/nbproject/* -!/CoreLibs/nbproject/project.xml -!/CoreLibs/nbproject/project.properties - -/Core/test/qa-functional/data/ - -/KeywordSearch/release/ -/KeywordSearch/build/ -/KeywordSearch/dist/ -/KeywordSearch/nbproject/* -!/KeywordSearch/nbproject/project.xml -!/KeywordSearch/nbproject/project.properties - -*/genfiles.properties -genfiles.properties - -/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties -/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties -/branding/build/ -/branding/dist/ -/branding/nbproject/* -!/branding/nbproject/project.xml -!/branding/nbproject/project.properties - -/test/input/* -!/test/input/notablehashes.txt-md5.idx -!/test/input/notablekeywords.xml -!/test/input/NSRL.txt-md5.idx -/test/output/* -!/test/output/gold -/test/script/output_dir_link.txt -/test/output/gold/tmp -/test/script/ScriptLog.txt -/test/script/__pycache__/ -/test/script/*.pyc -/test/script/DBDump-Diff.txt -/test/script/DBDump.txt -/test/script/SortedData-Diff.txt -/test/script/SortedData.txt -/test/script/myconfig.xml -/test/script/*/*.xml -/test/build/ -/test/dist/ -/test/nbproject/* - -!/Testing/nbproject/project.xml -!/Testing/nbproject/project.properties -*~ -/netbeans-plat -/docs/doxygen-user/user-docs -/jdiff-javadocs/* -/jdiff-logs/* -/gen_version.txt -hs_err_pid*.log - -.DS_Store -.*.swp -/Experimental/release/ -/ImageGallery/release/ -/thunderbirdparser/release/ -/RecentActivity/release/ -/CentralRepository/release/ -/Tika/release - -.idea/ -*.iml - -*.img -*.vhd -*.E01 +/dist/ +/build/ +/*/build/ +*/nbproject/private/* +/nbproject/private/* + +/Core/release/ +/Core/src/org/sleuthkit/autopsy/coreutils/Version.properties +/Core/src/org/sleuthkit/autopsy/casemodule/docs/QuickStart.html +/Core/src/org/sleuthkit/autopsy/casemodule/docs/screenshot.png +/Core/src/org/sleuthkit/autopsy/datamodel/ranges.csv +/Core/build/ +/Core/dist/ +/Core/nbproject/* +/Core/test/qa-functional/data/* +!/Core/nbproject/project.xml +!/Core/nbproject/project.properties + +/CoreLibs/release/ +/CoreLibs/build/ +/CoreLibs/dist/ +/CoreLibs/nbproject/* +!/CoreLibs/nbproject/project.xml +!/CoreLibs/nbproject/project.properties + +/Core/test/qa-functional/data/ + +/KeywordSearch/release/ +/KeywordSearch/build/ +/KeywordSearch/dist/ +/KeywordSearch/nbproject/* +!/KeywordSearch/nbproject/project.xml +!/KeywordSearch/nbproject/project.properties + +*/genfiles.properties +genfiles.properties + +/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties +/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties +/branding/build/ +/branding/dist/ +/branding/nbproject/* +!/branding/nbproject/project.xml +!/branding/nbproject/project.properties + +/test/input/* +!/test/input/notablehashes.txt-md5.idx +!/test/input/notablekeywords.xml +!/test/input/NSRL.txt-md5.idx +/test/output/* +!/test/output/gold +/test/script/output_dir_link.txt +/test/output/gold/tmp +/test/script/ScriptLog.txt +/test/script/__pycache__/ +/test/script/*.pyc +/test/script/DBDump-Diff.txt +/test/script/DBDump.txt +/test/script/SortedData-Diff.txt +/test/script/SortedData.txt +/test/script/myconfig.xml +/test/script/*/*.xml +/test/build/ +/test/dist/ +/test/nbproject/* + +!/Testing/nbproject/project.xml +!/Testing/nbproject/project.properties +*~ +/netbeans-plat +/docs/doxygen-user/user-docs +/jdiff-javadocs/* +/jdiff-logs/* +/gen_version.txt +hs_err_pid*.log + +.DS_Store +.*.swp +/Experimental/release/ +/ImageGallery/release/ +/thunderbirdparser/release/ +/RecentActivity/release/ +/CentralRepository/release/ +/Tika/release + +.idea/ +*.iml + +*.img +*.vhd +*.E01 + +/thirdparty/yara/yarabridge/yarabridge/x64/ +/thirdparty/yara/yarabridge/yarabridge.VC.db +/thirdparty/yara/yarabridge/yarabridge.VC.VC.opendb +/thirdparty/yara/yarabridge/x64/ +/thirdparty/yara/YaraWrapperTest/nbproject/private/ +/thirdparty/yara/YaraWrapperTest/build/ +/thirdparty/yara/YaraJNIWrapper/dist/ +/thirdparty/yara/YaraJNIWrapper/build/ +/thirdparty/yara/YaraJNIWrapper/nbproject/private/ +/thirdparty/yara/yarabridge/.vs/ + diff --git a/Core/build.xml b/Core/build.xml index 43715c6778..a9807f490b 100644 --- a/Core/build.xml +++ b/Core/build.xml @@ -98,6 +98,11 @@ + + + + + diff --git a/Core/nbproject/project.properties b/Core/nbproject/project.properties index 5646508ec5..d563edf2a5 100644 --- a/Core/nbproject/project.properties +++ b/Core/nbproject/project.properties @@ -121,6 +121,7 @@ file.reference.StixLib.jar=release\\modules\\ext\\StixLib.jar file.reference.threetenbp-1.3.3.jar=release\\modules\\ext\\threetenbp-1.3.3.jar file.reference.webp-imageio-sejda-0.1.0.jar=release\\modules\\ext\\webp-imageio-sejda-0.1.0.jar file.reference.xmpcore-5.1.3.jar=release\\modules\\ext\\xmpcore-5.1.3.jar +file.reference.YaraJNIWrapper.jar=release\\modules\\ext\\YaraJNIWrapper.jar file.reference.zookeeper-3.4.6.jar=release\\modules\\ext\\zookeeper-3.4.6.jar javac.source=11 javac.compilerargs=-Xlint -Xlint:-serial diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index c021ff6f04..bbcb967fa7 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -564,6 +564,10 @@ ext/jsoup-1.10.3.jar release\modules\ext\jsoup-1.10.3.jar + + ext/YaraJNIWrapper.jar + release/modules/ext/YaraJNIWrapper.jar + ext/grpc-context-1.19.0.jar release\modules\ext\grpc-context-1.19.0.jar diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDbManager.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDbManager.java index b0f288673b..d3a0e61ecf 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDbManager.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDbManager.java @@ -382,6 +382,31 @@ public class CentralRepoDbManager { CentralRepoDbUtil.setUseCentralRepo(true); saveNewCentralRepo(); } + + /** + * Set up a PostgresDb using the settings for the given database choice + * enum. + * + * @param choice Type of postgres DB to set up + * @throws CentralRepoException + */ + public void setupPostgresDb(CentralRepoDbChoice choice) throws CentralRepoException { + selectedDbChoice = choice; + DatabaseTestResult curStatus = testStatus(); + if (curStatus == DatabaseTestResult.DB_DOES_NOT_EXIST) { + createDb(); + curStatus = testStatus(); + } + + // the only successful setup status is tested ok + if (curStatus != DatabaseTestResult.TESTED_OK) { + throw new CentralRepoException("Unable to successfully create postgres database. Test failed with: " + curStatus); + } + + // if successfully got here, then save the settings + CentralRepoDbUtil.setUseCentralRepo(true); + saveNewCentralRepo(); + } /** * This method returns if changes to the central repository configuration diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresCentralRepoSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresCentralRepoSettings.java index 165d4ca6e0..dfb855a0b2 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresCentralRepoSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresCentralRepoSettings.java @@ -121,10 +121,12 @@ public final class PostgresCentralRepoSettings implements CentralRepoDbConnectiv * @return */ String getConnectionURL(boolean usePostgresDb) { - StringBuilder url = new StringBuilder(); - url.append(getJDBCBaseURI()); - url.append(getHost()); - url.append("/"); // NON-NLS + StringBuilder url = new StringBuilder() + .append(getJDBCBaseURI()) + .append(getHost()) + .append(":") // NON-NLS + .append(getPort()) + .append("/"); // NON-NLS if (usePostgresDb) { url.append("postgres"); // NON-NLS } else { @@ -153,7 +155,7 @@ public final class PostgresCentralRepoSettings implements CentralRepoDbConnectiv } catch (ClassNotFoundException | SQLException ex) { // TODO: Determine why a connection failure (ConnectionException) re-throws // the SQLException and does not print this log message? - LOGGER.log(Level.SEVERE, "Failed to acquire ephemeral connection to postgresql."); // NON-NLS + LOGGER.log(Level.SEVERE, "Failed to acquire ephemeral connection to postgresql.", ex); // NON-NLS conn = null; } return conn; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/Bundle.properties-MERGED index d4cca6c407..e95a759c4f 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/Bundle.properties-MERGED @@ -1,4 +1,10 @@ caseeventlistener.evidencetag=Evidence +CentralRepositoryNotificationDialog.bulletHeader=This data is used to: +CentralRepositoryNotificationDialog.bulletOne=Ignore common items (files, domains, and accounts) +CentralRepositoryNotificationDialog.bulletThree=Create personas that group accounts +CentralRepositoryNotificationDialog.bulletTwo=Identify where an item was previously seen +CentralRepositoryNotificationDialog.finalRemarks=To limit what is stored, use the Central Repository options panel. +CentralRepositoryNotificationDialog.header=Autopsy stores data about each case in its Central Repository. IngestEventsListener.ingestmodule.name=Central Repository IngestEventsListener.prevCaseComment.text=Previous Case: # {0} - typeName @@ -7,6 +13,3 @@ IngestEventsListener.prevCount.text=Number of previous {0}: {1} IngestEventsListener.prevExists.text=Previously Seen Devices (Central Repository) IngestEventsListener.prevTaggedSet.text=Previously Tagged As Notable (Central Repository) Installer.centralRepoUpgradeFailed.title=Central repository disabled -Installer.initialCreateSqlite.messageDesc=It will store information about all hashes and identifiers that you process. You can use this to ignore previously seen files and make connections between cases. -Installer.initialCreateSqlite.messageHeader=The Central Repository is not enabled. Would you like to enable it? -Installer.initialCreateSqlite.title=Enable Central Repository? diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CentralRepositoryNotificationDialog.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CentralRepositoryNotificationDialog.java new file mode 100755 index 0000000000..883d979c90 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CentralRepositoryNotificationDialog.java @@ -0,0 +1,73 @@ +/* + * Central Repository + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.centralrepository.eventlisteners; + +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.core.RuntimeProperties; +import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; +import org.sleuthkit.autopsy.coreutils.Version; + +/** + * Notifies new installations or old upgrades that the central repository will + * be enabled by default. + */ +public class CentralRepositoryNotificationDialog { + + /** + * This dialog should display iff the mode is RELEASE and the + * application is running with a GUI. + */ + static boolean shouldDisplay() { + return Version.getBuildType() == Version.Type.RELEASE + && RuntimeProperties.runningWithGUI(); + } + + /** + * Displays an informational modal dialog to the user, which is dismissed by + * pressing 'OK'. + */ + @NbBundle.Messages({ + "CentralRepositoryNotificationDialog.header=Autopsy stores data about each case in its Central Repository.", + "CentralRepositoryNotificationDialog.bulletHeader=This data is used to:", + "CentralRepositoryNotificationDialog.bulletOne=Ignore common items (files, domains, and accounts)", + "CentralRepositoryNotificationDialog.bulletTwo=Identify where an item was previously seen", + "CentralRepositoryNotificationDialog.bulletThree=Create personas that group accounts", + "CentralRepositoryNotificationDialog.finalRemarks=To limit what is stored, use the Central Repository options panel." + }) + static void display() { + assert shouldDisplay(); + + MessageNotifyUtil.Message.info( + "" + + "" + + "
" + + "

" + Bundle.CentralRepositoryNotificationDialog_header() + "

" + + "

" + Bundle.CentralRepositoryNotificationDialog_bulletHeader() + "

" + + "
    " + + "
  • " + Bundle.CentralRepositoryNotificationDialog_bulletOne() + "
    • " + + "
  • " + Bundle.CentralRepositoryNotificationDialog_bulletTwo() + "
    • " + + "
  • " + Bundle.CentralRepositoryNotificationDialog_bulletThree() + "
    • " + + "
" + + "

" + Bundle.CentralRepositoryNotificationDialog_finalRemarks() + "

" + + "
" + + "" + + "" + ); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/Installer.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/Installer.java index d4f0253cd2..b2ef0d437e 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/Installer.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/Installer.java @@ -25,14 +25,13 @@ import javax.swing.JOptionPane; import javax.swing.SwingUtilities; import org.openide.modules.ModuleInstall; import org.openide.util.NbBundle; -import org.openide.windows.WindowManager; -import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoDbChoice; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoDbManager; import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoException; import org.sleuthkit.autopsy.core.RuntimeProperties; +import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.ModuleSettings; -import org.sleuthkit.autopsy.coreutils.Version; /** * Adds/removes application event listeners responsible for adding data to the @@ -81,19 +80,10 @@ public class Installer extends ModuleInstall { * the org.sleuthkit.autopsy.core package when the already installed * Autopsy-Core module is restored (during application startup). */ - @NbBundle.Messages({ - "Installer.initialCreateSqlite.title=Enable Central Repository?", - "Installer.initialCreateSqlite.messageHeader=The Central Repository is not enabled. Would you like to enable it?", - "Installer.initialCreateSqlite.messageDesc=It will store information about all hashes and identifiers that you process. " - + "You can use this to ignore previously seen files and make connections between cases." - }) @Override public void restored() { addApplicationEventListeners(); - - if (Version.getBuildType() == Version.Type.RELEASE) { - setupDefaultCentralRepository(); - } + setupDefaultCentralRepository(); } /** @@ -107,9 +97,9 @@ public class Installer extends ModuleInstall { /** * Checks if the central repository has been set up and configured. If not, - * either offers to perform set up (running with a GUI) or does the set up - * unconditionally (not running with a GUI, e.g., in an automated ingest - * node). + * does the set up unconditionally. If the application is running with a + * GUI, a notification will be displayed to the user if the mode is RELEASE + * (in other words, developers are exempt from seeing the notification). */ private void setupDefaultCentralRepository() { Map centralRepoSettings = ModuleSettings.getConfigSettings("CentralRepository"); @@ -127,62 +117,30 @@ public class Installer extends ModuleInstall { ModuleSettings.setConfigSetting("CentralRepository", "initialized", "true"); } } - - // if central repository hasn't been previously initialized, initialize it - if (!initialized) { - // if running with a GUI, prompt the user - if (RuntimeProperties.runningWithGUI()) { - try { - SwingUtilities.invokeAndWait(() -> { - try { - String dialogText - = "" - + "
" - + "

" + NbBundle.getMessage(this.getClass(), "Installer.initialCreateSqlite.messageHeader") + "

" - + "

" + NbBundle.getMessage(this.getClass(), "Installer.initialCreateSqlite.messageDesc") + "

" - + "
" - + ""; - - if (JOptionPane.YES_OPTION == JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(), - dialogText, - NbBundle.getMessage(this.getClass(), "Installer.initialCreateSqlite.title"), - JOptionPane.YES_NO_OPTION)) { - - setupDefaultSqliteCentralRepo(); - } - } catch (CentralRepoException ex) { - logger.log(Level.SEVERE, "There was an error while initializing the central repository database", ex); - - doMessageBoxIfRunningInGUI(ex); - } - }); - } catch (InterruptedException | InvocationTargetException ex) { - logger.log(Level.SEVERE, "There was an error while running the swing utility invoke later while creating the central repository database", ex); - } - } // if no GUI, just initialize - else { - try { - setupDefaultSqliteCentralRepo(); - } catch (CentralRepoException ex) { - logger.log(Level.SEVERE, "There was an error while initializing the central repository database", ex); - - doMessageBoxIfRunningInGUI(ex); - } - } - - ModuleSettings.setConfigSetting("CentralRepository", "initialized", "true"); + + if(initialized) { + return; // Nothing to do } - } - /** - * Sets up a default single-user SQLite central repository. - * - * @throws CentralRepoException If there is an error setting up teh central - * repository. - */ - private void setupDefaultSqliteCentralRepo() throws CentralRepoException { - CentralRepoDbManager manager = new CentralRepoDbManager(); - manager.setupDefaultSqliteDb(); + if (CentralRepositoryNotificationDialog.shouldDisplay()) { + CentralRepositoryNotificationDialog.display(); + } + + try { + CentralRepoDbManager manager = new CentralRepoDbManager(); + if (UserPreferences.getIsMultiUserModeEnabled()) { + // Set up using existing multi-user settings. + manager.setupPostgresDb(CentralRepoDbChoice.POSTGRESQL_MULTIUSER); + } else { + manager.setupDefaultSqliteDb(); + } + } catch (CentralRepoException ex) { + logger.log(Level.SEVERE, "There was an error while initializing the central repository database", ex); + + doMessageBoxIfRunningInGUI(ex); + } + + ModuleSettings.setConfigSetting("CentralRepository", "initialized", "true"); } /** diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/RecentFilesSummary.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/RecentFilesSummary.java index 6c4601af88..651f5fe758 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/RecentFilesSummary.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/RecentFilesSummary.java @@ -28,10 +28,11 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; +import java.util.stream.Collectors; +import org.apache.commons.lang.StringUtils; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -93,16 +94,56 @@ public class RecentFilesSummary implements DefaultArtifactUpdateGovernor { return ARTIFACT_UPDATE_TYPE_IDS; } + /** + * Removes fileDetails entries with redundant paths, sorts by date + * descending and limits to the limit provided. + * + * @param fileDetails The file details list. + * @param limit The maximum number of entries to return. + * @return The sorted limited list with unique paths. + */ + private List getSortedLimited(List fileDetails, int limit) { + Map fileDetailsMap = fileDetails.stream() + .filter(details -> details != null) + .collect(Collectors.toMap( + d -> d.getPath().toUpperCase(), + d -> d, + (d1, d2) -> Long.compare(d1.getDateAsLong(), d2.getDateAsLong()) > 0 ? d1 : d2)); + + return fileDetailsMap.values().stream() + .sorted((a, b) -> -Long.compare(a.getDateAsLong(), b.getDateAsLong())) + .limit(limit) + .collect(Collectors.toList()); + } + + /** + * Returns a RecentFileDetails object as derived from the recent document + * artifact or null if no appropriate object can be made. + * + * @param artifact The artifact. + * @return The derived object or null if artifact is invalid. + */ + private RecentFileDetails getRecentlyOpenedDocument(BlackboardArtifact artifact) { + String path = DataSourceInfoUtilities.getStringOrNull(artifact, PATH_ATT); + Long lastOpened = DataSourceInfoUtilities.getLongOrNull(artifact, DATETIME_ATT); + + if (StringUtils.isBlank(path) || lastOpened == null || lastOpened == 0) { + return null; + } else { + return new RecentFileDetails(path, lastOpened); + } + } + /** * Return a list of the most recently opened documents based on the * TSK_RECENT_OBJECT artifact. * * @param dataSource The data source to query. - * @param maxCount The maximum number of results to return, pass 0 to get - * a list of all results. + * @param maxCount The maximum number of results to return, pass 0 to get a + * list of all results. * * @return A list RecentFileDetails representing the most recently opened - * documents or an empty list if none were found. + * documents or an empty list if none were found. * * @throws SleuthkitCaseProviderException * @throws TskCoreException @@ -112,36 +153,45 @@ public class RecentFilesSummary implements DefaultArtifactUpdateGovernor { return Collections.emptyList(); } - List artifactList - = DataSourceInfoUtilities.getArtifacts(provider.get(), - new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_RECENT_OBJECT), - dataSource, - DATETIME_ATT, - DataSourceInfoUtilities.SortOrder.DESCENDING, - maxCount); + throwOnNonPositiveCount(maxCount); - List fileDetails = new ArrayList<>(); - for (BlackboardArtifact artifact : artifactList) { - Long accessedTime = null; - String path = ""; + List details = provider.get().getBlackboard() + .getArtifacts(ARTIFACT_TYPE.TSK_RECENT_OBJECT.getTypeID(), dataSource.getId()).stream() + .map(art -> getRecentlyOpenedDocument(art)) + .filter(d -> d != null) + .collect(Collectors.toList()); - // Get all the attributes in one call. - List attributeList = artifact.getAttributes(); - for (BlackboardAttribute attribute : attributeList) { + return getSortedLimited(details, maxCount); + } - if (attribute.getAttributeType().equals(DATETIME_ATT)) { - accessedTime = attribute.getValueLong(); - } else if (attribute.getAttributeType().equals(PATH_ATT)) { - path = attribute.getValueString(); - } - } + /** + * Returns a RecentDownloadDetails object as derived from the recent + * download artifact or null if no appropriate object can be made. + * + * @param artifact The artifact. + * @return The derived object or null if artifact is invalid. + */ + private RecentDownloadDetails getRecentDownload(BlackboardArtifact artifact) { + Long accessedTime = DataSourceInfoUtilities.getLongOrNull(artifact, DATETIME_ACCESSED_ATT); + String domain = DataSourceInfoUtilities.getStringOrNull(artifact, DOMAIN_ATT); + String path = DataSourceInfoUtilities.getStringOrNull(artifact, PATH_ATT); - if (accessedTime != null && accessedTime != 0) { - fileDetails.add(new RecentFileDetails(path, accessedTime)); - } + if (StringUtils.isBlank(path) || accessedTime == null || accessedTime == 0) { + return null; + } else { + return new RecentDownloadDetails(path, accessedTime, domain); } + } - return fileDetails; + /** + * Throws an IllegalArgumentException if count is less than 1. + * + * @param count The count. + */ + private void throwOnNonPositiveCount(int count) { + if (count < 1) { + throw new IllegalArgumentException("Invalid count: value must be greater than 0."); + } } /** @@ -149,11 +199,11 @@ public class RecentFilesSummary implements DefaultArtifactUpdateGovernor { * artifact TSK_DATETIME_ACCESSED attribute. * * @param dataSource Data source to query. - * @param maxCount Maximum number of results to return, passing 0 will - * return all results. + * @param maxCount Maximum number of results to return, passing 0 will + * return all results. * * @return A list of RecentFileDetails objects or empty list if none were - * found. + * found. * * @throws TskCoreException * @throws SleuthkitCaseProviderException @@ -163,46 +213,23 @@ public class RecentFilesSummary implements DefaultArtifactUpdateGovernor { return Collections.emptyList(); } - List artifactList - = DataSourceInfoUtilities.getArtifacts(provider.get(), - new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_WEB_DOWNLOAD), - dataSource, - DATETIME_ACCESSED_ATT, - DataSourceInfoUtilities.SortOrder.DESCENDING, - maxCount); + throwOnNonPositiveCount(maxCount); - List fileDetails = new ArrayList<>(); - for (BlackboardArtifact artifact : artifactList) { - // Get all the attributes in one call. - Long accessedTime = null; - String domain = ""; - String path = ""; + List details = provider.get().getBlackboard() + .getArtifacts(ARTIFACT_TYPE.TSK_WEB_DOWNLOAD.getTypeID(), dataSource.getId()).stream() + .map(art -> getRecentDownload(art)) + .filter(d -> d != null) + .collect(Collectors.toList()); - List attributeList = artifact.getAttributes(); - for (BlackboardAttribute attribute : attributeList) { - - if (attribute.getAttributeType().equals(DATETIME_ACCESSED_ATT)) { - accessedTime = attribute.getValueLong(); - } else if (attribute.getAttributeType().equals(DOMAIN_ATT)) { - domain = attribute.getValueString(); - } else if (attribute.getAttributeType().equals(PATH_ATT)) { - path = attribute.getValueString(); - } - } - if (accessedTime != null && accessedTime != 0L) { - fileDetails.add(new RecentDownloadDetails(path, accessedTime, domain)); - } - } - - return fileDetails; + return getSortedLimited(details, maxCount); } /** * Returns a list of the most recent message attachments. * * @param dataSource Data source to query. - * @param maxCount Maximum number of results to return, passing 0 will - * return all results. + * @param maxCount Maximum number of results to return, passing 0 will + * return all results. * * @return A list of RecentFileDetails of the most recent attachments. * @@ -214,115 +241,72 @@ public class RecentFilesSummary implements DefaultArtifactUpdateGovernor { return Collections.emptyList(); } - if (maxCount < 0) { - throw new IllegalArgumentException("Invalid maxCount passed to getRecentAttachments, value must be equal to or greater than 0"); + throwOnNonPositiveCount(maxCount); + + SleuthkitCase skCase = provider.get(); + + List associatedArtifacts = skCase.getBlackboard() + .getArtifacts(ASSOCATED_OBJ_ART.getTypeID(), dataSource.getId()); + + List details = new ArrayList<>(); + for (BlackboardArtifact artifact : associatedArtifacts) { + RecentAttachmentDetails thisDetails = getRecentAttachment(artifact, skCase); + + if (thisDetails != null) { + details.add(thisDetails); + } } - return createListFromMap(buildAttachmentMap(dataSource), maxCount); + return getSortedLimited(details, maxCount); } /** - * Build a map of all of the message attachment sorted in date order. + * Creates a RecentAttachmentDetails object from the associated object + * artifact or null if no RecentAttachmentDetails object can be derived. * - * @param dataSource Data source to query. - * - * @return Returns a SortedMap of details objects returned in descending - * order. - * - * @throws SleuthkitCaseProviderException + * @param artifact The associated object artifact. + * @param skCase The current case. + * @return The derived object or null. * @throws TskCoreException */ - private SortedMap> buildAttachmentMap(DataSource dataSource) throws SleuthkitCaseProviderException, TskCoreException { - SleuthkitCase skCase = provider.get(); - TreeMap> sortedMap = new TreeMap<>(); - - List associatedArtifacts = skCase.getBlackboard().getArtifacts(ASSOCATED_OBJ_ART.getTypeID(), dataSource.getId()); - for (BlackboardArtifact artifact : associatedArtifacts) { - BlackboardAttribute attribute = artifact.getAttribute(ASSOCATED_ATT); - if (attribute == null) { - continue; - } - - BlackboardArtifact messageArtifact = skCase.getBlackboardArtifact(attribute.getValueLong()); - if (messageArtifact != null && isMessageArtifact(messageArtifact)) { - Content content = artifact.getParent(); - if (content instanceof AbstractFile) { - String sender; - Long date = null; - String path; - - BlackboardAttribute senderAttribute = messageArtifact.getAttribute(EMAIL_FROM_ATT); - if (senderAttribute != null) { - sender = senderAttribute.getValueString(); - } else { - sender = ""; - } - senderAttribute = messageArtifact.getAttribute(MSG_DATEIME_SENT_ATT); - if (senderAttribute != null) { - date = senderAttribute.getValueLong(); - } - - AbstractFile abstractFile = (AbstractFile) content; - - path = Paths.get(abstractFile.getParentPath(), abstractFile.getName()).toString(); - - if (date != null && date != 0) { - List list = sortedMap.get(date); - if (list == null) { - list = new ArrayList<>(); - sortedMap.put(date, list); - } - RecentAttachmentDetails details = new RecentAttachmentDetails(path, date, sender); - if (!list.contains(details)) { - list.add(details); - } - } - } - } - } - return sortedMap.descendingMap(); - } - - /** - * Create a list of detail objects from the given sorted map of the max - * size. - * - * @param sortedMap A Map of attachment details sorted by date. - * @param maxCount Maximum number of values to return. - * - * @return A list of the details of the most recent attachments or empty - * list if none where found. - */ - private List createListFromMap(SortedMap> sortedMap, int maxCount) { - List fileList = new ArrayList<>(); - - for (List mapList : sortedMap.values()) { - if (maxCount == 0 || fileList.size() + mapList.size() <= maxCount) { - fileList.addAll(mapList); - continue; - } - - if (maxCount == fileList.size()) { - break; - } - - for (RecentAttachmentDetails details : mapList) { - if (fileList.size() < maxCount) { - fileList.add(details); - } else { - break; - } - } + private RecentAttachmentDetails getRecentAttachment(BlackboardArtifact artifact, SleuthkitCase skCase) throws TskCoreException { + // get associated artifact or return no result + BlackboardAttribute attribute = artifact.getAttribute(ASSOCATED_ATT); + if (attribute == null) { + return null; } - return fileList; + // get associated message artifact if exists or return no result + BlackboardArtifact messageArtifact = skCase.getBlackboardArtifact(attribute.getValueLong()); + if (messageArtifact == null || !isMessageArtifact(messageArtifact)) { + return null; + } + + // get abstract file if exists or return no result + Content content = artifact.getParent(); + if (!(content instanceof AbstractFile)) { + return null; + } + + AbstractFile abstractFile = (AbstractFile) content; + + // get the path, sender, and date + String path = Paths.get(abstractFile.getParentPath(), abstractFile.getName()).toString(); + String sender = DataSourceInfoUtilities.getStringOrNull(messageArtifact, EMAIL_FROM_ATT); + Long date = DataSourceInfoUtilities.getLongOrNull(messageArtifact, MSG_DATEIME_SENT_ATT); + + if (date == null || date == 0 || StringUtils.isBlank(path)) { + return null; + } else { + return new RecentAttachmentDetails(path, date, sender); + } } /** * Is the given artifact a message. * * @param nodeArtifact An artifact that might be a message. Must not be - * null. + * null. * * @return True if the given artifact is a message artifact */ @@ -330,6 +314,7 @@ public class RecentFilesSummary implements DefaultArtifactUpdateGovernor { final int artifactTypeID = nodeArtifact.getArtifactTypeID(); return artifactTypeID == ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID() || artifactTypeID == ARTIFACT_TYPE.TSK_MESSAGE.getTypeID(); + } /** @@ -391,8 +376,8 @@ public class RecentFilesSummary implements DefaultArtifactUpdateGovernor { /** * Constructor for files with just a path and date. * - * @param path File path. - * @param date File access date\time in seconds with java epoch. + * @param path File path. + * @param date File access date\time in seconds with java epoch. * @param webDomain The webdomain from which the file was downloaded. */ RecentDownloadDetails(String path, long date, String webDomain) { @@ -404,7 +389,7 @@ public class RecentFilesSummary implements DefaultArtifactUpdateGovernor { * Returns the web domain. * * @return The web domain or empty string if not available or - * applicable. + * applicable. */ public String getWebDomain() { return webDomain; @@ -422,10 +407,10 @@ public class RecentFilesSummary implements DefaultArtifactUpdateGovernor { * Constructor for recent download files which have a path, date and * domain value. * - * @param path File path. - * @param date File crtime. + * @param path File path. + * @param date File crtime. * @param sender The sender of the message from which the file was - * attached. + * attached. */ RecentAttachmentDetails(String path, long date, String sender) { super(path, date); @@ -436,7 +421,7 @@ public class RecentFilesSummary implements DefaultArtifactUpdateGovernor { * Return the sender of the attached file. * * @return The sender of the attached file or empty string if not - * available. + * available. */ public String getSender() { return sender; diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/UserActivitySummary.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/UserActivitySummary.java index 16da6f5c4b..b7efda6e29 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/UserActivitySummary.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/UserActivitySummary.java @@ -137,7 +137,7 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { return (a.getProgramName() == null ? "" : a.getProgramName()) .compareToIgnoreCase((b.getProgramName() == null ? "" : b.getProgramName())); }; - + private static final Set ARTIFACT_UPDATE_TYPE_IDS = new HashSet<>(Arrays.asList( ARTIFACT_TYPE.TSK_WEB_SEARCH_QUERY.getTypeID(), ARTIFACT_TYPE.TSK_MESSAGE.getTypeID(), @@ -172,9 +172,9 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { * is designed with unit testing in mind since mocked dependencies can be * utilized. * - * @param provider The object providing the current SleuthkitCase. + * @param provider The object providing the current SleuthkitCase. * @param translationService The translation service. - * @param logger The logger to use. + * @param logger The logger to use. */ public UserActivitySummary( SleuthkitCaseProvider provider, @@ -206,7 +206,7 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { * Gets a list of recent domains based on the datasource. * * @param dataSource The datasource to query for recent domains. - * @param count The max count of items to return. + * @param count The max count of items to return. * * @return The list of items retrieved from the database. * @@ -242,12 +242,12 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { * Creates a TopDomainsResult from data or null if no visit date exists * within DOMAIN_WINDOW_MS of mostRecentMs. * - * @param domain The domain. - * @param visits The number of visits. + * @param domain The domain. + * @param visits The number of visits. * @param mostRecentMs The most recent visit of any domain. * * @return The TopDomainsResult or null if no visits to this domain within - * 30 days of mostRecentMs. + * 30 days of mostRecentMs. */ private TopDomainsResult getDomainsResult(String domain, List visits, long mostRecentMs) { long visitCount = 0; @@ -280,9 +280,8 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { * @param dataSource The datasource. * * @return A tuple where the first value is the latest web history accessed - * date in milliseconds and the second value maps normalized - * (lowercase; trimmed) domain names to when those domains were - * visited. + * date in milliseconds and the second value maps normalized (lowercase; + * trimmed) domain names to when those domains were visited. * * @throws TskCoreException * @throws SleuthkitCaseProviderException @@ -349,7 +348,7 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { * @param artifact The artifact. * * @return The TopWebSearchResult or null if the search string or date - * accessed cannot be determined. + * accessed cannot be determined. */ private static TopWebSearchResult getWebSearchResult(BlackboardArtifact artifact) { String searchString = DataSourceInfoUtilities.getStringOrNull(artifact, TYPE_TEXT); @@ -364,11 +363,10 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { * term. * * @param dataSource The data source. - * @param count The maximum number of records to be shown (must be > - * 0). + * @param count The maximum number of records to be shown (must be > 0). * * @return The list of most recent web searches where most recent search - * appears first. + * appears first. * * @throws * org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException @@ -386,21 +384,22 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { .getArtifacts(ARTIFACT_TYPE.TSK_WEB_SEARCH_QUERY.getTypeID(), dataSource.getId()); // group by search string (case insensitive) - Collection> resultGroups = webSearchArtifacts + Collection resultGroups = webSearchArtifacts .stream() // get items where search string and date is not null .map(UserActivitySummary::getWebSearchResult) // remove null records .filter(result -> result != null) - // get these messages grouped by search to string - .collect(Collectors.groupingBy((result) -> result.getSearchString().toUpperCase())) + // get the latest message for each search string + .collect(Collectors.toMap( + (result) -> result.getSearchString().toUpperCase(), + result -> result, + (result1, result2) -> TOP_WEBSEARCH_RESULT_DATE_COMPARE.compare(result1, result2) >= 0 ? result1 : result2)) .values(); // get the most recent date for each search term List results = resultGroups .stream() - // get the most recent access per search type - .map((list) -> list.stream().max(TOP_WEBSEARCH_RESULT_DATE_COMPARE).get()) // get most recent searches first .sorted(TOP_WEBSEARCH_RESULT_DATE_COMPARE.reversed()) .limit(count) @@ -424,7 +423,7 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { * @param original The original text. * * @return The translated text or null if no translation can be determined - * or exists. + * or exists. */ private String getTranslationOrNull(String original) { if (!translationService.hasProvider() || StringUtils.isBlank(original)) { @@ -448,15 +447,34 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { return translated; } + /** + * Gives the most recent TopDeviceAttachedResult. If one is null, the other + * is returned. + * + * @param r1 A result. + * @param r2 Another result. + * @return The most recent one with a non-null date. + */ + private TopDeviceAttachedResult getMostRecentDevice(TopDeviceAttachedResult r1, TopDeviceAttachedResult r2) { + if (r2.getDateAccessed() == null) { + return r1; + } + + if (r1.getDateAccessed() == null) { + return r2; + } + + return r1.getDateAccessed().compareTo(r2.getDateAccessed()) >= 0 ? r1 : r2; + } + /** * Retrieves most recent devices used by most recent date attached. * * @param dataSource The data source. - * @param count The maximum number of records to be shown (must be > - * 0). + * @param count The maximum number of records to be shown (must be > 0). * * @return The list of most recent devices attached where most recent device - * attached appears first. + * attached appears first. * * @throws * org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException @@ -469,7 +487,7 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { return Collections.emptyList(); } - return DataSourceInfoUtilities.getArtifacts(caseProvider.get(), TYPE_DEVICE_ATTACHED, + Collection results = DataSourceInfoUtilities.getArtifacts(caseProvider.get(), TYPE_DEVICE_ATTACHED, dataSource, TYPE_DATETIME, DataSourceInfoUtilities.SortOrder.DESCENDING, 0) .stream() .map(artifact -> { @@ -482,9 +500,14 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { }) // remove Root Hub identifier .filter(result -> { - return result.getDeviceModel() == null + return result.getDeviceId() == null + || result.getDeviceModel() == null || !DEVICE_EXCLUDE_LIST.contains(result.getDeviceModel().trim().toUpperCase()); }) + .collect(Collectors.toMap(result -> result.getDeviceId(), result -> result, (r1, r2) -> getMostRecentDevice(r1, r2))) + .values(); + + return results.stream() .limit(count) .collect(Collectors.toList()); } @@ -495,7 +518,7 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { * @param artifact The artifact. * * @return The TopAccountResult or null if the account type or message date - * cannot be determined. + * cannot be determined. */ private static TopAccountResult getMessageAccountResult(BlackboardArtifact artifact) { String type = DataSourceInfoUtilities.getStringOrNull(artifact, TYPE_MESSAGE_TYPE); @@ -509,12 +532,12 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { * Obtains a TopAccountResult from a blackboard artifact. The date is * maximum of any found dates for attribute types provided. * - * @param artifact The artifact. + * @param artifact The artifact. * @param messageType The type of message this is. - * @param dateAttrs The date attribute types. + * @param dateAttrs The date attribute types. * * @return The TopAccountResult or null if the account type or max date are - * not provided. + * not provided. */ private static TopAccountResult getAccountResult(BlackboardArtifact artifact, String messageType, BlackboardAttribute.Type... dateAttrs) { String type = messageType; @@ -538,11 +561,10 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { * sent. * * @param dataSource The data source. - * @param count The maximum number of records to be shown (must be > - * 0). + * @param count The maximum number of records to be shown (must be > 0). * * @return The list of most recent accounts used where the most recent - * account by last message sent occurs first. + * account by last message sent occurs first. * * @throws * org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException @@ -585,18 +607,19 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { Stream allResults = Stream.concat(messageResults, Stream.concat(emailResults, calllogResults)); // get them grouped by account type - Collection> groupedResults = allResults + Collection groupedResults = allResults // remove null records .filter(result -> result != null) - // get these messages grouped by account type - .collect(Collectors.groupingBy(TopAccountResult::getAccountType)) + // get these messages grouped by account type and get the most recent of each type + .collect(Collectors.toMap( + result -> result.getAccountType(), + result -> result, + (result1, result2) -> TOP_ACCOUNT_RESULT_DATE_COMPARE.compare(result1, result2) >= 0 ? result1 : result2)) .values(); // get account type sorted by most recent date return groupedResults .stream() - // get the most recent access per account type - .map((accountGroup) -> accountGroup.stream().max(TOP_ACCOUNT_RESULT_DATE_COMPARE).get()) // get most recent accounts accessed .sorted(TOP_ACCOUNT_RESULT_DATE_COMPARE.reversed()) // limit to count @@ -608,7 +631,7 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { /** * Determines a short folder name if any. Otherwise, returns empty string. * - * @param strPath The string path. + * @param strPath The string path. * @param applicationName The application name. * * @return The short folder name or empty string if not found. @@ -659,7 +682,7 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { if (StringUtils.startsWithIgnoreCase(path, WINDOWS_PREFIX)) { return null; } - + Integer count = DataSourceInfoUtilities.getIntOrNull(artifact, TYPE_COUNT); Long longCount = (count == null) ? null : (long) count; @@ -696,7 +719,7 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { * @param long2 Second possibly null long. * * @return Returns the compare value: 1,0,-1 favoring the higher non-null - * value. + * value. */ private static int nullableCompare(Long long1, Long long2) { if (long1 == null && long2 == null) { @@ -721,7 +744,6 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { return longNum != null && longNum > 0; } - /** * Retrieves the top programs results for the given data source limited to * the count provided as a parameter. The highest run times are at the top @@ -731,12 +753,12 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { * be ignored and all items will be returned. * * @param dataSource The datasource. If the datasource is null, an empty - * list will be returned. - * @param count The number of results to return. This value must be > 0 - * or an IllegalArgumentException will be thrown. + * list will be returned. + * @param count The number of results to return. This value must be > 0 or + * an IllegalArgumentException will be thrown. * * @return The sorted list and limited to the count if last run or run count - * information is available on any item. + * information is available on any item. * * @throws SleuthkitCaseProviderException * @throws TskCoreException @@ -759,7 +781,9 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { // The value will be a TopProgramsResult with the max run times // and most recent last run date for each program name / program path pair. .collect(Collectors.toMap( - res -> Pair.of(res.getProgramName(), res.getProgramPath()), + res -> Pair.of( + res.getProgramName() == null ? null : res.getProgramName().toUpperCase(), + res.getProgramPath() == null ? null : res.getProgramPath().toUpperCase()), res -> res, (res1, res2) -> { return new TopProgramsResult( @@ -852,10 +876,10 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { /** * Main constructor. * - * @param deviceId The device id. + * @param deviceId The device id. * @param dateAccessed The date last attached. - * @param deviceMake The device make. - * @param deviceModel The device model. + * @param deviceMake The device make. + * @param deviceModel The device model. */ public TopDeviceAttachedResult(String deviceId, Date dateAccessed, String deviceMake, String deviceModel) { this.deviceId = deviceId; @@ -906,7 +930,7 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { * Main constructor. * * @param accountType The account type. - * @param lastAccess The date the account was last accessed. + * @param lastAccess The date the account was last accessed. */ public TopAccountResult(String accountType, Date lastAccess) { this.accountType = accountType; @@ -940,9 +964,9 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { /** * Describes a top domain result. * - * @param domain The domain. + * @param domain The domain. * @param visitTimes The number of times it was visited. - * @param lastVisit The date of the last visit. + * @param lastVisit The date of the last visit. */ public TopDomainsResult(String domain, Long visitTimes, Date lastVisit) { this.domain = domain; @@ -987,7 +1011,7 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { * * @param programName The name of the program. * @param programPath The path of the program. - * @param runTimes The number of runs. + * @param runTimes The number of runs. */ TopProgramsResult(String programName, String programPath, Long runTimes, Date lastRun) { this.programName = programName; diff --git a/Core/src/org/sleuthkit/autopsy/guicomponeontutils/AbstractCheckboxListItem.java b/Core/src/org/sleuthkit/autopsy/guicomponeontutils/AbstractCheckboxListItem.java new file mode 100755 index 0000000000..df0a00888f --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/guicomponeontutils/AbstractCheckboxListItem.java @@ -0,0 +1,51 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.guicomponeontutils; + +import javax.swing.Icon; +import org.sleuthkit.autopsy.guiutils.CheckBoxJList; + +/** + * An abstract implementation of CheckBoxJList.CheckboxListItem so that + * implementing classes have default implementation. + */ +public abstract class AbstractCheckboxListItem implements CheckBoxJList.CheckboxListItem { + + private boolean checked = false; + + @Override + public boolean isChecked() { + return checked; + } + + @Override + public void setChecked(boolean checked) { + this.checked = checked; + } + + @Override + public boolean hasIcon() { + return false; + } + + @Override + public Icon getIcon() { + return null; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/guiutils/CheckBoxJList.java b/Core/src/org/sleuthkit/autopsy/guiutils/CheckBoxJList.java index ab17187c4b..f59db1259f 100755 --- a/Core/src/org/sleuthkit/autopsy/guiutils/CheckBoxJList.java +++ b/Core/src/org/sleuthkit/autopsy/guiutils/CheckBoxJList.java @@ -32,8 +32,10 @@ import javax.swing.ListSelectionModel; /** * A JList that renders the list items as check boxes. + * + * @param An object that implements CheckboxListItem */ -final class CheckBoxJList extends JList { +public final class CheckBoxJList extends JList { private static final long serialVersionUID = 1L; @@ -42,7 +44,7 @@ final class CheckBoxJList extends JLis * a checkbox in CheckBoxJList. * */ - interface CheckboxListItem { + public interface CheckboxListItem { /** * Returns the checkbox state. @@ -83,7 +85,7 @@ final class CheckBoxJList extends JLis /** * Construct a new JCheckBoxList. */ - CheckBoxJList() { + public CheckBoxJList() { initalize(); } @@ -134,12 +136,15 @@ final class CheckBoxJList extends JLis checkbox.setSelected(value.isChecked()); checkbox.setBackground(list.getBackground()); checkbox.setEnabled(list.isEnabled()); + checkbox.setOpaque(list.isOpaque()); label.setText(value.getDisplayName()); label.setEnabled(list.isEnabled()); + label.setOpaque(list.isOpaque()); if (value.hasIcon()) { label.setIcon(value.getIcon()); } + setOpaque(list.isOpaque()); setEnabled(list.isEnabled()); return this; } diff --git a/Core/src/org/sleuthkit/autopsy/images/yara_32.png b/Core/src/org/sleuthkit/autopsy/images/yara_32.png new file mode 100755 index 0000000000..9fdb23052a Binary files /dev/null and b/Core/src/org/sleuthkit/autopsy/images/yara_32.png differ diff --git a/Core/src/org/sleuthkit/autopsy/modules/drones/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/drones/Bundle.properties-MERGED index bf61ad9be0..cd35320f71 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/drones/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/modules/drones/Bundle.properties-MERGED @@ -1,6 +1,6 @@ DATExtractor_process_message=Processing DJI DAT file: %s DATFileExtractor_Extractor_Name=DAT File Extractor -DroneIngestModule_Description=Analyzes files generated by drones. -DroneIngestModule_Name=Drone Analyzer +DroneIngestModule_Description=Analyzes files generated by some DJI drones. +DroneIngestModule_Name=DJI Drone Analyzer # {0} - AbstractFileName DroneIngestModule_process_start=Started {0} diff --git a/Core/src/org/sleuthkit/autopsy/modules/drones/DroneIngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/drones/DroneIngestModuleFactory.java index 1213ffedd5..0e8f7107a5 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/drones/DroneIngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/modules/drones/DroneIngestModuleFactory.java @@ -33,8 +33,8 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; public class DroneIngestModuleFactory extends IngestModuleFactoryAdapter { @Messages({ - "DroneIngestModule_Name=Drone Analyzer", - "DroneIngestModule_Description=Analyzes files generated by drones." + "DroneIngestModule_Name=DJI Drone Analyzer", + "DroneIngestModule_Description=Analyzes files generated by some DJI drones." }) /** diff --git a/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/Bundle.properties index 0036d4dd6f..15698d736c 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/Bundle.properties @@ -1,3 +1,4 @@ ILeappAnalyzerIngestModule.init.exception.msg=Unable to find {0}. ILeappAnalyzerIngestModule.processing.file=Processing file {0} -ILeappAnalyzerIngestModule.parsing.file=Parsing file {0} \ No newline at end of file +ILeappAnalyzerIngestModule.parsing.file=Parsing file {0} +ILeappAnalyzerIngestModule.processing.filesystem=Processing filesystem \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/Bundle.properties-MERGED index b4f350f478..b4e8226a91 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/Bundle.properties-MERGED @@ -8,6 +8,7 @@ ILeappAnalyzerIngestModule.iLeapp.cancelled=iLeapp run was canceled ILeappAnalyzerIngestModule.init.exception.msg=Unable to find {0}. ILeappAnalyzerIngestModule.processing.file=Processing file {0} ILeappAnalyzerIngestModule.parsing.file=Parsing file {0} +ILeappAnalyzerIngestModule.processing.filesystem=Processing filesystem ILeappAnalyzerIngestModule.report.name=iLeapp Html Report ILeappAnalyzerIngestModule.requires.windows=iLeapp module requires windows. ILeappAnalyzerIngestModule.running.iLeapp=Running iLeapp diff --git a/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/ILeappAnalyzerIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/ILeappAnalyzerIngestModule.java index f859919bae..abec90e6c0 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/ILeappAnalyzerIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/ILeappAnalyzerIngestModule.java @@ -18,8 +18,10 @@ */ package org.sleuthkit.autopsy.modules.ileappanalyzer; +import java.io.BufferedReader; import java.io.File; import java.io.FileNotFoundException; +import java.io.FileReader; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Files; @@ -32,14 +34,17 @@ import java.util.Locale; import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.commons.io.FilenameUtils; import org.openide.modules.InstalledFileLocator; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import static org.sleuthkit.autopsy.casemodule.Case.getCurrentCase; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.ExecUtil; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.PlatformUtil; +import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.ingest.DataSourceIngestModule; import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProcessTerminator; import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; @@ -50,6 +55,7 @@ import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.LocalFilesDataSource; +import org.sleuthkit.datamodel.ReadContentInputStream; import org.sleuthkit.datamodel.TskCoreException; /** @@ -61,7 +67,9 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { private static final String MODULE_NAME = ILeappAnalyzerModuleFactory.getModuleName(); private static final String ILEAPP = "iLeapp"; //NON-NLS + private static final String ILEAPP_FS = "fs_"; //NON-NLS private static final String ILEAPP_EXECUTABLE = "ileapp.exe";//NON-NLS + private static final String ILEAPP_PATHS_FILE = "iLeapp_paths.txt"; //NON-NLS private File iLeappExecutable; @@ -87,7 +95,7 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { try { iLeappFileProcessor = new ILeappFileProcessor(); - } catch (IOException | IngestModuleException ex) { + } catch (IOException | IngestModuleException | NoCurrentCaseException ex) { throw new IngestModuleException(Bundle.ILeappAnalyzerIngestModule_error_ileapp_file_processor_init(), ex); } @@ -112,58 +120,49 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { @Override public ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) { - if (!(context.getDataSource() instanceof LocalFilesDataSource)) { - return ProcessResult.OK; + Case currentCase = Case.getCurrentCase(); + Path tempOutputPath = Paths.get(currentCase.getTempDirectory(), ILEAPP, ILEAPP_FS + dataSource.getId()); + try { + Files.createDirectories(tempOutputPath); + } catch (IOException ex) { + logger.log(Level.SEVERE, String.format("Error creating iLeapp output directory %s", tempOutputPath.toString()), ex); + return ProcessResult.ERROR; + } + + List iLeappPathsToProcess = new ArrayList<>(); + ProcessBuilder iLeappCommand = buildiLeappListCommand(tempOutputPath); + try { + int result = ExecUtil.execute(iLeappCommand, new DataSourceIngestModuleProcessTerminator(context, true)); + if (result != 0) { + logger.log(Level.SEVERE, String.format("Error when trying to execute iLeapp program getting file paths to search for result is %d", result)); + return ProcessResult.ERROR; + } + iLeappPathsToProcess = loadIleappPathFile(tempOutputPath); + } catch (IOException ex) { + logger.log(Level.SEVERE, String.format("Error when trying to execute iLeapp program getting file paths to search"), ex); + return ProcessResult.ERROR; } statusHelper.progress(Bundle.ILeappAnalyzerIngestModule_starting_iLeapp(), 0); - List iLeappFilesToProcess = findiLeappFilesToProcess(dataSource); + List iLeappFilesToProcess = new ArrayList<>(); - statusHelper.switchToDeterminate(iLeappFilesToProcess.size()); + if (!(context.getDataSource() instanceof LocalFilesDataSource)) { + extractFilesFromImage(dataSource, iLeappPathsToProcess, tempOutputPath); + statusHelper.switchToDeterminate(iLeappFilesToProcess.size()); + processILeappFs(dataSource, currentCase, statusHelper, tempOutputPath.toString()); + } else { + iLeappFilesToProcess = findiLeappFilesToProcess(dataSource); + statusHelper.switchToDeterminate(iLeappFilesToProcess.size()); - Integer filesProcessedCount = 0; - - Case currentCase = Case.getCurrentCase(); - for (AbstractFile iLeappFile : iLeappFilesToProcess) { - - String currentTime = new SimpleDateFormat("yyyy-MM-dd HH-mm-ss z", Locale.US).format(System.currentTimeMillis());//NON-NLS - Path moduleOutputPath = Paths.get(currentCase.getModuleDirectory(), ILEAPP, currentTime); - try { - Files.createDirectories(moduleOutputPath); - } catch (IOException ex) { - logger.log(Level.SEVERE, String.format("Error creating iLeapp output directory %s", moduleOutputPath.toString()), ex); - return ProcessResult.ERROR; + Integer filesProcessedCount = 0; + for (AbstractFile iLeappFile : iLeappFilesToProcess) { + processILeappFile(dataSource, currentCase, statusHelper, filesProcessedCount, iLeappFile); + filesProcessedCount++; } - - statusHelper.progress(NbBundle.getMessage(this.getClass(), "ILeappAnalyzerIngestModule.processing.file", iLeappFile.getName()), filesProcessedCount); - ProcessBuilder iLeappCommand = buildiLeappCommand(moduleOutputPath, iLeappFile.getLocalAbsPath(), iLeappFile.getNameExtension()); - try { - int result = ExecUtil.execute(iLeappCommand, new DataSourceIngestModuleProcessTerminator(context, true)); - if (result != 0) { - // ignore if there is an error and continue to try and process the next file if there is one - continue; - } - - addILeappReportToReports(moduleOutputPath, currentCase); - - } catch (IOException ex) { - logger.log(Level.SEVERE, String.format("Error when trying to execute iLeapp program against file %s", iLeappFile.getLocalAbsPath()), ex); - return ProcessResult.ERROR; - } - - if (context.dataSourceIngestIsCancelled()) { - logger.log(Level.INFO, "ILeapp Analyser ingest module run was canceled"); //NON-NLS - return ProcessResult.OK; - } - - ProcessResult fileProcessorResult = iLeappFileProcessor.processFiles(dataSource, moduleOutputPath, iLeappFile); - - if (fileProcessorResult == ProcessResult.ERROR) { - return ProcessResult.ERROR; - } - - filesProcessedCount++; + // Process the logical image as a fs in iLeapp to make sure this is not a logical fs that was added + extractFilesFromImage(dataSource, iLeappPathsToProcess, tempOutputPath); + processILeappFs(dataSource, currentCase, statusHelper, tempOutputPath.toString()); } IngestMessage message = IngestMessage.createMessage(IngestMessage.MessageType.DATA, @@ -173,6 +172,99 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { return ProcessResult.OK; } + /** + * Process each tar/zip file that is found in a logical image that contains xLeapp data + * @param dataSource Datasource where the file has been found + * @param currentCase current case + * @param statusHelper Progress bar for messages to show user + * @param filesProcessedCount count that is incremented for progress bar + * @param iLeappFile abstract file that will be processed + */ + private void processILeappFile(Content dataSource, Case currentCase, DataSourceIngestModuleProgress statusHelper, int filesProcessedCount, + AbstractFile iLeappFile) { + String currentTime = new SimpleDateFormat("yyyy-MM-dd HH-mm-ss z", Locale.US).format(System.currentTimeMillis());//NON-NLS + Path moduleOutputPath = Paths.get(currentCase.getModuleDirectory(), ILEAPP, currentTime); + try { + Files.createDirectories(moduleOutputPath); + } catch (IOException ex) { + logger.log(Level.SEVERE, String.format("Error creating iLeapp output directory %s", moduleOutputPath.toString()), ex); + return; + } + + statusHelper.progress(NbBundle.getMessage(this.getClass(), "ILeappAnalyzerIngestModule.processing.file", iLeappFile.getName()), filesProcessedCount); + ProcessBuilder iLeappCommand = buildiLeappCommand(moduleOutputPath, iLeappFile.getLocalAbsPath(), iLeappFile.getNameExtension()); + try { + int result = ExecUtil.execute(iLeappCommand, new DataSourceIngestModuleProcessTerminator(context, true)); + if (result != 0) { + logger.log(Level.WARNING, String.format("Error when trying to execute iLeapp program getting file paths to search for result is %d", result)); + return; + } + + addILeappReportToReports(moduleOutputPath, currentCase); + + } catch (IOException ex) { + logger.log(Level.SEVERE, String.format("Error when trying to execute iLeapp program against file %s", iLeappFile.getLocalAbsPath()), ex); + return; + } + + if (context.dataSourceIngestIsCancelled()) { + logger.log(Level.INFO, "ILeapp Analyser ingest module run was canceled"); //NON-NLS + return; + } + + ProcessResult fileProcessorResult = iLeappFileProcessor.processFiles(dataSource, moduleOutputPath, iLeappFile); + + if (fileProcessorResult == ProcessResult.ERROR) { + return; + } + } + + /** + * Process extracted files from a disk image using xLeapp + * @param dataSource Datasource where the file has been found + * @param currentCase current case + * @param statusHelper Progress bar for messages to show user + * @param directoryToProcess + */ + private void processILeappFs(Content dataSource, Case currentCase, DataSourceIngestModuleProgress statusHelper, String directoryToProcess) { + String currentTime = new SimpleDateFormat("yyyy-MM-dd HH-mm-ss z", Locale.US).format(System.currentTimeMillis());//NON-NLS + Path moduleOutputPath = Paths.get(currentCase.getModuleDirectory(), ILEAPP, currentTime); + try { + Files.createDirectories(moduleOutputPath); + } catch (IOException ex) { + logger.log(Level.SEVERE, String.format("Error creating iLeapp output directory %s", moduleOutputPath.toString()), ex); + return; + } + + statusHelper.progress(NbBundle.getMessage(this.getClass(), "ILeappAnalyzerIngestModule.processing.filesystem")); + ProcessBuilder iLeappCommand = buildiLeappCommand(moduleOutputPath, directoryToProcess, "fs"); + try { + int result = ExecUtil.execute(iLeappCommand, new DataSourceIngestModuleProcessTerminator(context, true)); + if (result != 0) { + logger.log(Level.WARNING, String.format("Error when trying to execute iLeapp program getting file paths to search for result is %d", result)); + return; + } + + addILeappReportToReports(moduleOutputPath, currentCase); + + } catch (IOException ex) { + logger.log(Level.SEVERE, String.format("Error when trying to execute iLeapp program against file system"), ex); + return; + } + + if (context.dataSourceIngestIsCancelled()) { + logger.log(Level.INFO, "ILeapp Analyser ingest module run was canceled"); //NON-NLS + return; + } + + ProcessResult fileProcessorResult = iLeappFileProcessor.processFileSystem(dataSource, moduleOutputPath); + + if (fileProcessorResult == ProcessResult.ERROR) { + return; + } + + } + /** * Find the files that will be processed by the iLeapp program * @@ -197,17 +289,24 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { List iLeappFilesToProcess = new ArrayList<>(); for (AbstractFile iLeappFile : iLeappFiles) { if (((iLeappFile.getLocalAbsPath() != null) - && (!iLeappFile.getNameExtension().isEmpty() && (!iLeappFile.isVirtual()))) - && ((iLeappFile.getName().toLowerCase().contains(".zip") || (iLeappFile.getName().toLowerCase().contains(".tar"))) - || iLeappFile.getName().toLowerCase().contains(".tgz"))) { - iLeappFilesToProcess.add(iLeappFile); - + && (!iLeappFile.getNameExtension().isEmpty() && (!iLeappFile.isVirtual()))) + && ((iLeappFile.getName().toLowerCase().contains(".zip") || (iLeappFile.getName().toLowerCase().contains(".tar"))) + || iLeappFile.getName().toLowerCase().contains(".tgz"))) { + iLeappFilesToProcess.add(iLeappFile); + } } return iLeappFilesToProcess; } + /** + * Build the command to run xLeapp + * @param moduleOutputPath output path for xLeapp + * @param sourceFilePath path where the xLeapp file is + * @param iLeappFileSystem type of file to process tar/zip/fs + * @return process to run + */ private ProcessBuilder buildiLeappCommand(Path moduleOutputPath, String sourceFilePath, String iLeappFileSystemType) { ProcessBuilder processBuilder = buildProcessWithRunAsInvoker( @@ -221,10 +320,26 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { return processBuilder; } + /** + * Command to run xLeapp using the path option + * @param moduleOutputPath path where the file paths output will reside + * @return process to run + */ + private ProcessBuilder buildiLeappListCommand(Path moduleOutputPath) { + + ProcessBuilder processBuilder = buildProcessWithRunAsInvoker( + "\"" + iLeappExecutable + "\"", //NON-NLS + "-p" + ); + processBuilder.redirectError(moduleOutputPath.resolve("iLeapp_paths_error.txt").toFile()); //NON-NLS + processBuilder.redirectOutput(moduleOutputPath.resolve("iLeapp_paths.txt").toFile()); //NON-NLS + return processBuilder; + } + static private ProcessBuilder buildProcessWithRunAsInvoker(String... commandLine) { ProcessBuilder processBuilder = new ProcessBuilder(commandLine); /* - * Add an environment variable to force log2timeline/psort to run with + * Add an environment variable to force iLeapp to run with * the same permissions Autopsy uses. */ processBuilder.environment().put("__COMPAT_LAYER", "RunAsInvoker"); //NON-NLS @@ -248,13 +363,18 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { private void addILeappReportToReports(Path iLeappOutputDir, Case currentCase) { List allIndexFiles = new ArrayList<>(); - try (Stream walk = Files.walk(iLeappOutputDir)) { + try (Stream walk = Files.walk(iLeappOutputDir)) { allIndexFiles = walk.map(x -> x.toString()) .filter(f -> f.toLowerCase().endsWith("index.html")).collect(Collectors.toList()); if (!allIndexFiles.isEmpty()) { - currentCase.addReport(allIndexFiles.get(0), MODULE_NAME, Bundle.ILeappAnalyzerIngestModule_report_name()); + // Check for existance of directory that holds report data if does not exist then report contains no data + String filePath = FilenameUtils.getFullPathNoEndSeparator(allIndexFiles.get(0)); + File dataFilesDir = new File(Paths.get(filePath, "_TSV Exports").toString()); + if (dataFilesDir.exists()) { + currentCase.addReport(allIndexFiles.get(0), MODULE_NAME, Bundle.ILeappAnalyzerIngestModule_report_name()); + } } } catch (IOException | UncheckedIOException | TskCoreException ex) { @@ -264,4 +384,129 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { } + /* + * Reads the iLeapp paths file to get the paths that we want to extract + * + * @param moduleOutputPath path where the file paths output will reside + */ + private List loadIleappPathFile(Path moduleOutputPath) throws FileNotFoundException, IOException { + List iLeappPathsToProcess = new ArrayList<>(); + + Path filePath = Paths.get(moduleOutputPath.toString(), ILEAPP_PATHS_FILE); + + try (BufferedReader reader = new BufferedReader(new FileReader(filePath.toString()))) { + String line = reader.readLine(); + while (line != null) { + if (line.contains("path list generation") || line.length() < 2) { + line = reader.readLine(); + continue; + } + iLeappPathsToProcess.add(line.trim()); + line = reader.readLine(); + } + } + + return iLeappPathsToProcess; + } + + /** + * Extract files from a disk image to process with xLeapp + * @param dataSource Datasource of the image + * @param iLeappPathsToProcess List of paths to extract content from + * @param moduleOutputPath path to write content to + */ + private void extractFilesFromImage(Content dataSource, List iLeappPathsToProcess, Path moduleOutputPath) { + FileManager fileManager = getCurrentCase().getServices().getFileManager(); + + for (String fullFilePath : iLeappPathsToProcess) { + + if (context.dataSourceIngestIsCancelled()) { + logger.log(Level.INFO, "ILeapp Analyser ingest module run was canceled"); //NON-NLS + break; + } + + String ffp = fullFilePath.replaceAll("\\*", "%"); + ffp = FilenameUtils.normalize(ffp, true); + String fileName = FilenameUtils.getName(ffp); + String filePath = FilenameUtils.getPath(ffp); + + List iLeappFiles = new ArrayList<>(); + try { + if (filePath.isEmpty()) { + iLeappFiles = fileManager.findFiles(dataSource, fileName); //NON-NLS + } else { + iLeappFiles = fileManager.findFiles(dataSource, fileName, filePath); //NON-NLS + } + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "No files found to process"); //NON-NLS + return; + } + + for (AbstractFile iLeappFile : iLeappFiles) { + Path parentPath = Paths.get(moduleOutputPath.toString(), iLeappFile.getParentPath()); + File fileParentPath = new File(parentPath.toString()); + + extractFileToOutput(dataSource, iLeappFile, fileParentPath, parentPath); + } + } + } + + /** + * Create path and file from datasource in temp + * @param dataSource datasource of the image + * @param iLeappFile abstract file to write out + * @param fileParentPath parent file path + * @param parentPath parent file + */ + private void extractFileToOutput(Content dataSource, AbstractFile iLeappFile, File fileParentPath, Path parentPath) { + if (fileParentPath.exists()) { + if (!iLeappFile.isDir()) { + writeiLeappFile(dataSource, iLeappFile, fileParentPath.toString()); + } else { + try { + Files.createDirectories(Paths.get(parentPath.toString(), iLeappFile.getName())); + } catch (IOException ex) { + logger.log(Level.INFO, String.format("Error creating iLeapp output directory %s", parentPath.toString()), ex); + } + } + } else { + try { + Files.createDirectories(parentPath); + } catch (IOException ex) { + logger.log(Level.INFO, String.format("Error creating iLeapp output directory %s", parentPath.toString()), ex); + } + if (!iLeappFile.isDir()) { + writeiLeappFile(dataSource, iLeappFile, fileParentPath.toString()); + } else { + try { + Files.createDirectories(Paths.get(parentPath.toString(), iLeappFile.getName())); + } catch (IOException ex) { + logger.log(Level.INFO, String.format("Error creating iLeapp output directory %s", parentPath.toString()), ex); + } + } + } + } + + /** + * Write out file to output + * @param dataSource datasource of disk image + * @param iLeappFile acstract file to write out + * @param parentPath path to write file to + */ + private void writeiLeappFile(Content dataSource, AbstractFile iLeappFile, String parentPath) { + String fileName = iLeappFile.getName().replace(":", "-"); + if (!fileName.matches(".") && !fileName.matches("..") && !fileName.toLowerCase().endsWith("-slack")) { + Path filePath = Paths.get(parentPath, fileName); + File localFile = new File(filePath.toString()); + try { + ContentUtils.writeToFile(iLeappFile, localFile, context::dataSourceIngestIsCancelled); + } catch (ReadContentInputStream.ReadContentInputStreamException ex) { + logger.log(Level.WARNING, String.format("Error reading file '%s' (id=%d).", + iLeappFile.getName(), iLeappFile.getId()), ex); //NON-NLS + } catch (IOException ex) { + logger.log(Level.WARNING, String.format("Error writing file local file '%s' (id=%d).", + filePath.toString(), iLeappFile.getId()), ex); //NON-NLS + } + } + } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/ILeappFileProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/ILeappFileProcessor.java index cb5ab516a6..058b0aa28d 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/ILeappFileProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/ILeappFileProcessor.java @@ -44,6 +44,7 @@ import javax.xml.parsers.ParserConfigurationException; import org.apache.commons.io.FilenameUtils; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; @@ -76,12 +77,16 @@ public final class ILeappFileProcessor { private final Map tsvFileArtifactComments; private final Map>> tsvFileAttributes; - public ILeappFileProcessor() throws IOException, IngestModuleException { + Blackboard blkBoard; + + public ILeappFileProcessor() throws IOException, IngestModuleException, NoCurrentCaseException { this.tsvFiles = new HashMap<>(); this.tsvFileArtifacts = new HashMap<>(); this.tsvFileArtifactComments = new HashMap<>(); this.tsvFileAttributes = new HashMap<>(); + blkBoard = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard(); + configExtractor(); loadConfigFile(); @@ -110,6 +115,19 @@ public final class ILeappFileProcessor { return ProcessResult.OK; } + public ProcessResult processFileSystem(Content dataSource, Path moduleOutputPath) { + + try { + List iLeappTsvOutputFiles = findTsvFiles(moduleOutputPath); + processiLeappFiles(iLeappTsvOutputFiles, dataSource); + } catch (IOException | IngestModuleException ex) { + logger.log(Level.SEVERE, String.format("Error trying to process iLeapp output files in directory %s. ", moduleOutputPath.toString()), ex); //NON-NLS + return ProcessResult.ERROR; + } + + return ProcessResult.OK; + } + /** * Find the tsv files in the iLeapp output directory and match them to files * we know we want to process and return the list to process those files. @@ -124,7 +142,7 @@ public final class ILeappFileProcessor { .filter(f -> f.toLowerCase().endsWith(".tsv")).collect(Collectors.toList()); for (String tsvFile : allTsvFiles) { - if (tsvFiles.containsKey(FilenameUtils.getName(tsvFile))) { + if (tsvFiles.containsKey(FilenameUtils.getName(tsvFile.toLowerCase()))) { foundTsvFiles.add(tsvFile); } } @@ -160,7 +178,41 @@ public final class ILeappFileProcessor { processFile(iLeappFile, attrList, fileName, artifactType, bbartifacts, iLeappImageFile); } catch (TskCoreException ex) { - // check this + throw new IngestModuleException(String.format("Error getting Blackboard Artifact Type for %s", tsvFileArtifacts.get(fileName)), ex); + } + } + + } + + if (!bbartifacts.isEmpty()) { + postArtifacts(bbartifacts); + } + + } + + /** + * Process the iLeapp files that were found that match the xml mapping file + * + * @param iLeappFilesToProcess List of files to process + * @param iLeappImageFile Abstract file to create artifact for + * + * @throws FileNotFoundException + * @throws IOException + */ + private void processiLeappFiles(List iLeappFilesToProcess, Content dataSource) throws FileNotFoundException, IOException, IngestModuleException { + List bbartifacts = new ArrayList<>(); + + for (String iLeappFileName : iLeappFilesToProcess) { + String fileName = FilenameUtils.getName(iLeappFileName); + File iLeappFile = new File(iLeappFileName); + if (tsvFileAttributes.containsKey(fileName)) { + List> attrList = tsvFileAttributes.get(fileName); + try { + BlackboardArtifact.Type artifactType = Case.getCurrentCase().getSleuthkitCase().getArtifactType(tsvFileArtifacts.get(fileName)); + + processFile(iLeappFile, attrList, fileName, artifactType, bbartifacts, dataSource); + + } catch (TskCoreException ex) { throw new IngestModuleException(String.format("Error getting Blackboard Artifact Type for %s", tsvFileArtifacts.get(fileName)), ex); } } @@ -174,7 +226,8 @@ public final class ILeappFileProcessor { } private void processFile(File iLeappFile, List> attrList, String fileName, BlackboardArtifact.Type artifactType, - List bbartifacts, AbstractFile iLeappImageFile) throws FileNotFoundException, IOException, IngestModuleException { + List bbartifacts, Content dataSource) throws FileNotFoundException, IOException, IngestModuleException, + TskCoreException { try (BufferedReader reader = new BufferedReader(new FileReader(iLeappFile))) { String line = reader.readLine(); // Check first line, if it is null then no heading so nothing to match to, close and go to next file. @@ -183,8 +236,8 @@ public final class ILeappFileProcessor { line = reader.readLine(); while (line != null) { Collection bbattributes = processReadLine(line, columnNumberToProcess, fileName); - if (!bbattributes.isEmpty()) { - BlackboardArtifact bbartifact = createArtifactWithAttributes(artifactType.getTypeID(), iLeappImageFile, bbattributes); + if (!bbattributes.isEmpty() && !blkBoard.artifactExists(dataSource, BlackboardArtifact.ARTIFACT_TYPE.fromID(artifactType.getTypeID()), bbattributes)) { + BlackboardArtifact bbartifact = createArtifactWithAttributes(artifactType.getTypeID(), dataSource, bbattributes); if (bbartifact != null) { bbartifacts.add(bbartifact); } @@ -234,8 +287,8 @@ public final class ILeappFileProcessor { } - private void checkAttributeType(Collection bbattributes, String attrType, String[] columnValues, Integer columnNumber, BlackboardAttribute.Type attributeType, - String fileName) { + private void checkAttributeType(Collection bbattributes, String attrType, String[] columnValues, Integer columnNumber, BlackboardAttribute.Type attributeType, + String fileName) { if (attrType.matches("STRING")) { bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, columnValues[columnNumber])); } else if (attrType.matches("INTEGER")) { @@ -340,7 +393,7 @@ public final class ILeappFileProcessor { for (int i = 0; i < nlist.getLength(); i++) { NamedNodeMap nnm = nlist.item(i).getAttributes(); - tsvFiles.put(nnm.getNamedItem("filename").getNodeValue(), nnm.getNamedItem("description").getNodeValue()); + tsvFiles.put(nnm.getNamedItem("filename").getNodeValue().toLowerCase(), nnm.getNamedItem("description").getNodeValue()); } @@ -393,19 +446,20 @@ public final class ILeappFileProcessor { } } - /** - * Generic method for creating a blackboard artifact with attributes - * - * @param type is a blackboard.artifact_type enum to determine - * which type the artifact should be - * @param abstractFile is the AbstractFile object that needs to have the - * artifact added for it - * @param bbattributes is the collection of blackboard attributes that - * need to be added to the artifact after the - * artifact has been created - * - * @return The newly-created artifact, or null on error - */ + + /** + * Generic method for creating a blackboard artifact with attributes + * + * @param type is a blackboard.artifact_type enum to determine which + * type the artifact should be + * @param abstractFile is the AbstractFile object that needs to have the + * artifact added for it + * @param bbattributes is the collection of blackboard attributes that need + * to be added to the artifact after the artifact has + * been created + * + * @return The newly-created artifact, or null on error + */ private BlackboardArtifact createArtifactWithAttributes(int type, AbstractFile abstractFile, Collection bbattributes) { try { BlackboardArtifact bbart = abstractFile.newArtifact(type); @@ -417,6 +471,30 @@ public final class ILeappFileProcessor { return null; } + /** + * Generic method for creating a blackboard artifact with attributes + * + * @param type is a blackboard.artifact_type enum to determine which + * type the artifact should be + * @param datasource is the Content object that needs to have the artifact + * added for it + * @param bbattributes is the collection of blackboard attributes that need + * to be added to the artifact after the artifact has + * been created + * + * @return The newly-created artifact, or null on error + */ + private BlackboardArtifact createArtifactWithAttributes(int type, Content dataSource, Collection bbattributes) { + try { + BlackboardArtifact bbart = dataSource.newArtifact(type); + bbart.addAttributes(bbattributes); + return bbart; + } catch (TskException ex) { + logger.log(Level.WARNING, Bundle.ILeappFileProcessor_error_creating_new_artifacts(), ex); //NON-NLS + } + return null; + } + /** * Method to post a list of BlackboardArtifacts to the blackboard. * diff --git a/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/ileap-artifact-attribute-reference.xml b/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/ileap-artifact-attribute-reference.xml index 0959cbdcc4..a4169395aa 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/ileap-artifact-attribute-reference.xml +++ b/Core/src/org/sleuthkit/autopsy/modules/ileappanalyzer/ileap-artifact-attribute-reference.xml @@ -47,6 +47,15 @@ + + + + + + + + + @@ -120,6 +129,13 @@ + + + + + + + @@ -189,6 +205,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -207,15 +253,61 @@ - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -248,6 +340,19 @@ + + + + + + + + + + + + + @@ -288,6 +393,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -302,6 +437,18 @@ + + + + + + + + + + + + @@ -317,6 +464,18 @@ + + + + + + + + + + + + @@ -433,6 +592,102 @@ --> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/yara/Bundle.properties-MERGED new file mode 100755 index 0000000000..8f898ba127 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/Bundle.properties-MERGED @@ -0,0 +1,5 @@ +Yara_Module_Description=With the YARA ingest module you use YARA rule files to search files for textual or binary patterns. +Yara_Module_Name=YARA +YaraIngestModule_no_ruleSets=Unable to run YARA ingest, list of YARA rule sets was empty. +YaraIngestModule_windows_error_msg=The YARA ingest module is only available on 64bit Windows. +YaraIngestModule_yarac_not_found=Unable to compile YARA rules files. Unable to find executable at. diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/YaraIngestHelper.java b/Core/src/org/sleuthkit/autopsy/modules/yara/YaraIngestHelper.java new file mode 100755 index 0000000000..4b71a62351 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/YaraIngestHelper.java @@ -0,0 +1,258 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.yara; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import org.openide.modules.InstalledFileLocator; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.coreutils.ExecUtil; +import org.sleuthkit.autopsy.ingest.IngestModule; +import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; +import org.sleuthkit.autopsy.modules.yara.rules.RuleSet; +import org.sleuthkit.autopsy.modules.yara.rules.RuleSetManager; +import org.sleuthkit.autopsy.yara.YaraJNIWrapper; +import org.sleuthkit.autopsy.yara.YaraWrapperException; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.BlackboardArtifact; +import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_YARA_HIT; +import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME; +import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_RULE; +import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Methods for scanning files for yara rule matches. + */ +final class YaraIngestHelper { + + private static final String YARA_DIR = "yara"; + private static final String YARA_C_EXE = "yarac64.exe"; + private static final String MODULE_NAME = YaraIngestModuleFactory.getModuleName(); + + private YaraIngestHelper() { + } + + /** + * Uses the yarac tool to compile the rules in the given rule sets. + * + * @param ruleSetNames List of names of the selected rule sets. + * @param tempDir Path of the directory to put the compiled rule files. + * + * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException + */ + static void compileRules(List ruleSetNames, Path outputDir) throws IngestModuleException { + if (ruleSetNames == null || ruleSetNames.isEmpty()) { + throw new IngestModule.IngestModuleException(Bundle.YaraIngestModule_no_ruleSets()); + } + + // Find javac + File exeFile = InstalledFileLocator.getDefault().locate( + Paths.get(YARA_DIR, YARA_C_EXE).toString(), + YaraIngestModule.class.getPackage().getName(), false); + + if (exeFile == null) { + throw new IngestModuleException(Bundle.YaraIngestModule_yarac_not_found()); + } + + for (RuleSet set : getRuleSetsForNames(ruleSetNames)) { + compileRuleSet(set, outputDir, exeFile); + } + } + + /** + * Scan the given AbstractFile for yara rule matches from the rule sets in + * the given directory creating a blackboard artifact for each matching + * rule. + * + * The baseDirectory should contain a series of directories one for each + * rule set. + * + * @param file The file to scan. + * @param baseRuleSetDirectory Base directory for the compiled rule sets. + * + * @throws TskCoreException + */ + static List scanFileForMatches(AbstractFile file, File baseRuleSetDirectory, byte[] fileData, int fileDataSize, int timeout) throws TskCoreException, YaraWrapperException { + List artifacts = new ArrayList<>(); + + File[] ruleSetDirectories = baseRuleSetDirectory.listFiles(); + for (File ruleSetDirectory : ruleSetDirectories) { + + List ruleMatches = YaraIngestHelper.scanFileForMatches(fileData, fileDataSize, ruleSetDirectory, timeout); + if (!ruleMatches.isEmpty()) { + artifacts.addAll(YaraIngestHelper.createArtifact(file, ruleSetDirectory.getName(), ruleMatches)); + } + } + + return artifacts; + } + + /** + * + * @param file The Abstract File being processed. + * @param baseRuleSetDirectory Base directory of the compiled rule sets. + * @param localFile Local copy of file. + * @param timeout Yara file scan timeout in seconds. + * + * @return + * + * @throws TskCoreException + * @throws YaraWrapperException + */ + static List scanFileForMatches(AbstractFile file, File baseRuleSetDirectory, File localFile, int timeout) throws TskCoreException, YaraWrapperException { + List artifacts = new ArrayList<>(); + + File[] ruleSetDirectories = baseRuleSetDirectory.listFiles(); + for (File ruleSetDirectory : ruleSetDirectories) { + List ruleMatches = YaraIngestHelper.scanFileForMatch(localFile, ruleSetDirectory, timeout); + if (!ruleMatches.isEmpty()) { + artifacts.addAll(YaraIngestHelper.createArtifact(file, ruleSetDirectory.getName(), ruleMatches)); + } + } + + return artifacts; + } + + /** + * Scan the given file byte array for rule matches using the YaraJNIWrapper + * API. + * + * @param fileBytes + * @param ruleSetDirectory + * + * @return List of rules that match from the given file from the given rule + * set. Empty list is returned if no matches where found. + * + * @throws TskCoreException + */ + private static List scanFileForMatches(byte[] fileBytes, int fileSize, File ruleSetDirectory, int timeout) throws YaraWrapperException { + List matchingRules = new ArrayList<>(); + + File[] ruleSetCompiledFileList = ruleSetDirectory.listFiles(); + + for (File ruleFile : ruleSetCompiledFileList) { + matchingRules.addAll(YaraJNIWrapper.findRuleMatch(ruleFile.getAbsolutePath(), fileBytes, fileSize, timeout)); + } + + return matchingRules; + } + + private static List scanFileForMatch(File scanFile, File ruleSetDirectory, int timeout) throws YaraWrapperException { + List matchingRules = new ArrayList<>(); + + File[] ruleSetCompiledFileList = ruleSetDirectory.listFiles(); + + for (File ruleFile : ruleSetCompiledFileList) { + matchingRules.addAll(YaraJNIWrapper.findRuleMatchFile(ruleFile.getAbsolutePath(), scanFile.getAbsolutePath(), timeout)); + } + + return matchingRules; + } + + /** + * Create a list of Blackboard Artifacts, one for each matching rule. + * + * @param abstractFile File to add artifact to. + * @param ruleSetName Name rule set with matching rule. + * @param matchingRules Matching rule. + * + * @return List of artifacts or empty list if none were found. + * + * @throws TskCoreException + */ + private static List createArtifact(AbstractFile abstractFile, String ruleSetName, List matchingRules) throws TskCoreException { + List artifacts = new ArrayList<>(); + for (String rule : matchingRules) { + BlackboardArtifact artifact = abstractFile.newArtifact(TSK_YARA_HIT); + List attributes = new ArrayList<>(); + + attributes.add(new BlackboardAttribute(TSK_SET_NAME, MODULE_NAME, ruleSetName)); + attributes.add(new BlackboardAttribute(TSK_RULE, MODULE_NAME, rule)); + + artifact.addAttributes(attributes); + artifacts.add(artifact); + } + return artifacts; + } + + @NbBundle.Messages({ + "YaraIngestModule_yarac_not_found=Unable to compile YARA rules files. Unable to find executable at.", + "YaraIngestModule_no_ruleSets=Unable to run YARA ingest, list of YARA rule sets was empty." + }) + + /** + * Compiles the rule files in the given rule set. + * + * The compiled rule files are created in outputDir\RuleSetName. + * + * @param set RuleSet for which to compile files. + * @param outputDir Output directory for the compiled rule files. + * @param yarac yarac executeable file. + * + * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException + */ + static private void compileRuleSet(RuleSet set, Path outputDir, File yarac) throws IngestModuleException { + File tempFolder = Paths.get(outputDir.toString(), set.getName()).toFile(); + if (!tempFolder.exists()) { + tempFolder.mkdir(); + } + + List fileList = set.getRuleFiles(); + for (File file : fileList) { + List commandList = new ArrayList<>(); + commandList.add(String.format("\"%s\"", yarac.toString())); + commandList.add(String.format("\"%s\"", file.toString())); + commandList.add(String.format("\"%s\"", Paths.get(tempFolder.getAbsolutePath(), "compiled_" + file.getName()))); + + ProcessBuilder builder = new ProcessBuilder(commandList); + try { + ExecUtil.execute(builder); + } catch (SecurityException | IOException ex) { + throw new IngestModuleException(String.format("Failed to compile Yara rules file", file.toString()), ex); + } + + } + } + + /** + * Returns a list of RuleSet objects for the given list of RuleSet names. + * + * @param names List of RuleSet names. + * + * @return List of RuleSet or empty list if none of the names matched + * existing rules. + */ + private static List getRuleSetsForNames(List names) { + List ruleSetList = new ArrayList<>(); + + RuleSetManager manager = new RuleSetManager(); + for (RuleSet set : manager.getRuleSetList()) { + if (names.contains(set.getName())) { + ruleSetList.add(set); + } + } + + return ruleSetList; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/YaraIngestJobSettings.java b/Core/src/org/sleuthkit/autopsy/modules/yara/YaraIngestJobSettings.java new file mode 100755 index 0000000000..b115bba90f --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/YaraIngestJobSettings.java @@ -0,0 +1,106 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.yara; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; +import org.sleuthkit.autopsy.modules.yara.rules.RuleSet; + +/** + * IngestJobSettings for the YARA ingest module. + */ +public final class YaraIngestJobSettings implements IngestModuleIngestJobSettings { + + private static final long serialVersionUID = 1L; + + private List selectedRuleSetNames; + private boolean onlyExecutableFiles; + + // Default constructor. + YaraIngestJobSettings() { + onlyExecutableFiles = true; + selectedRuleSetNames = new ArrayList<>(); + } + + /** + * Constructor. + * + * @param selected List of selected rules. + * @param onlyExecutableFiles Process only executable files. + */ + public YaraIngestJobSettings(List selected, boolean onlyExecutableFiles) { + this.selectedRuleSetNames = new ArrayList<>(); + + for (RuleSet set : selected) { + selectedRuleSetNames.add(set.getName()); + } + + this.onlyExecutableFiles = onlyExecutableFiles; + } + + /** + * Return the list of rule name sets that were selected in the ingest + * settings panel. + * + * @return List of selected RuleSet names. + */ + public List getSelectedRuleSetNames() { + return Collections.unmodifiableList(selectedRuleSetNames); + } + + /** + * Set the list of selected rule names. + * + * @param selected List of selected rule Sets. + */ + void setSelectedRuleSetNames(List selected) { + this.selectedRuleSetNames = new ArrayList<>(); + for (RuleSet set : selected) { + selectedRuleSetNames.add(set.getName()); + } + } + + /** + * Process only executable Files. + * + * @return If true the ingest module should process only executable files, + * if false process all files. + */ + public boolean onlyExecutableFiles() { + return onlyExecutableFiles; + } + + /** + * Set whether to process only executable files or all files. + * + * @param onlyExecutableFiles True if the ingest module should only process + * executable files. + */ + void setOnlyExecuteableFile(boolean onlyExecutableFiles) { + this.onlyExecutableFiles = onlyExecutableFiles; + } + + @Override + public long getVersionNumber() { + return serialVersionUID; + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/YaraIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/yara/YaraIngestModule.java new file mode 100755 index 0000000000..c8deff0936 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/YaraIngestModule.java @@ -0,0 +1,224 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.yara; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Level; +import org.apache.commons.lang3.RandomStringUtils; +import org.openide.util.NbBundle.Messages; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.PlatformUtil; +import org.sleuthkit.autopsy.datamodel.ContentUtils; +import org.sleuthkit.autopsy.ingest.FileIngestModuleAdapter; +import org.sleuthkit.autopsy.ingest.IngestJobContext; +import org.sleuthkit.autopsy.ingest.IngestModule; +import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; +import org.sleuthkit.autopsy.yara.YaraWrapperException; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Blackboard; +import org.sleuthkit.datamodel.Blackboard.BlackboardException; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskData; + +/** + * An ingest module that runs the yara against the given files. + * + */ +public class YaraIngestModule extends FileIngestModuleAdapter { + + // 15MB + private static final int FILE_SIZE_THRESHOLD_MB = 100; + private static final int FILE_SIZE_THRESHOLD_BYTE = FILE_SIZE_THRESHOLD_MB * 1024 * 1024; + private static final int YARA_SCAN_TIMEOUT_SEC = 30 * 60 * 60; // 30 minutes. + + private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); + private final static Logger logger = Logger.getLogger(YaraIngestModule.class.getName()); + private static final String YARA_DIR = "yara"; + private static final Map pathsByJobId = new ConcurrentHashMap<>(); + private static final String RULESET_DIR = "RuleSets"; + + private final YaraIngestJobSettings settings; + + private IngestJobContext context = null; + private Long jobId; + + /** + * Constructor. + * + * @param settings + */ + YaraIngestModule(YaraIngestJobSettings settings) { + this.settings = settings; + } + + @Messages({ + "YaraIngestModule_windows_error_msg=The YARA ingest module is only available on 64bit Windows.",}) + + @Override + public void startUp(IngestJobContext context) throws IngestModuleException { + this.context = context; + this.jobId = context.getJobId(); + + if (!PlatformUtil.isWindowsOS() || !PlatformUtil.is64BitOS()) { + throw new IngestModule.IngestModuleException(Bundle.YaraIngestModule_windows_error_msg()); + } + + if (refCounter.incrementAndGet(jobId) == 1) { + // compile the selected rules & put into temp folder based on jobID + Path tempDir = getTempDirectory(jobId); + Path tempRuleSetDir = Paths.get(tempDir.toString(), RULESET_DIR); + if(!tempRuleSetDir.toFile().exists()) { + tempRuleSetDir.toFile().mkdir(); + } + + YaraIngestHelper.compileRules(settings.getSelectedRuleSetNames(), tempRuleSetDir); + } + } + + @Override + public void shutDown() { + if (context != null && refCounter.decrementAndGet(jobId) == 0) { + // do some clean up. + Path jobPath = pathsByJobId.get(jobId); + if (jobPath != null) { + jobPath.toFile().delete(); + pathsByJobId.remove(jobId); + } + } + } + + @Override + public ProcessResult process(AbstractFile file) { + + if (settings.onlyExecutableFiles()) { + String extension = file.getNameExtension(); + if (!extension.equals("exe")) { + return ProcessResult.OK; + } + } + + // Skip the file if its 0 in length or a directory. + if (file.getSize() == 0 || + file.isDir() || + file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) { + return ProcessResult.OK; + } + + try { + List artifacts = new ArrayList<>(); + File ruleSetsDir = Paths.get(getTempDirectory(jobId).toString(), RULESET_DIR).toFile(); + + // If the file size is less than FILE_SIZE_THRESHOLD_BYTE read the file + // into a buffer, else make a local copy of the file. + if(file.getSize() < FILE_SIZE_THRESHOLD_BYTE) { + byte[] fileBuffer = new byte[(int)file.getSize()]; + + int dataRead = file.read(fileBuffer, 0, file.getSize()); + if(dataRead != 0) { + artifacts.addAll( YaraIngestHelper.scanFileForMatches(file, ruleSetsDir, fileBuffer, dataRead, YARA_SCAN_TIMEOUT_SEC)); + } + } else { + File tempCopy = createLocalCopy(file); + artifacts.addAll( YaraIngestHelper.scanFileForMatches(file, ruleSetsDir, tempCopy, YARA_SCAN_TIMEOUT_SEC)); + tempCopy.delete(); + } + + if(!artifacts.isEmpty()) { + Blackboard blackboard = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard(); + blackboard.postArtifacts(artifacts, YaraIngestModuleFactory.getModuleName()); + } + + } catch (BlackboardException | NoCurrentCaseException | IngestModuleException | TskCoreException | YaraWrapperException ex) { + logger.log(Level.SEVERE, String.format("YARA ingest module failed to process file id %d", file.getId()), ex); + return ProcessResult.ERROR; + } catch(IOException ex) { + logger.log(Level.SEVERE, String.format("YARA ingest module failed to make a local copy of given file id %d", file.getId()), ex); + return ProcessResult.ERROR; + } + + return ProcessResult.OK; + } + + /** + * Return the temp directory for this jobId. If the folder does not exit it + * will be created. + * + * @param jobId The current jobId + * + * @return The path of the temporary directory for the given jobId. + * + * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException + */ + private synchronized Path getTempDirectory(long jobId) throws IngestModuleException { + Path jobPath = pathsByJobId.get(jobId); + if (jobPath != null) { + return jobPath; + } + + Path baseDir; + try { + baseDir = Paths.get(Case.getCurrentCaseThrows().getTempDirectory(), YARA_DIR); + } catch (NoCurrentCaseException ex) { + throw new IngestModuleException("Failed to create YARA ingest model temp directory, no open case.", ex); + } + + // Make the base yara directory, as needed + if (!baseDir.toFile().exists()) { + baseDir.toFile().mkdirs(); + } + + String randomDirName = String.format("%s_%d", RandomStringUtils.randomAlphabetic(8), jobId); + jobPath = Paths.get(baseDir.toString(), randomDirName); + jobPath.toFile().mkdir(); + + pathsByJobId.put(jobId, jobPath); + + return jobPath; + } + + /** + * Create a local copy of the given AbstractFile. + * + * @param file AbstractFile to make a copy of. + * + * @return A File object representation of the local copy. + * + * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException + * @throws IOException + */ + protected File createLocalCopy(AbstractFile file) throws IngestModuleException, IOException { + String tempFileName = RandomStringUtils.randomAlphabetic(15) + file.getId() + ".temp"; + + File tempFile = Paths.get(getTempDirectory(context.getJobId()).toString(), tempFileName).toFile(); + ContentUtils.writeToFile(file, tempFile, context::dataSourceIngestIsCancelled); + + return tempFile; + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/YaraIngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/yara/YaraIngestModuleFactory.java new file mode 100755 index 0000000000..1abf8aaa0f --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/YaraIngestModuleFactory.java @@ -0,0 +1,92 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.yara; + +import java.util.ArrayList; +import org.openide.util.NbBundle.Messages; +import org.openide.util.lookup.ServiceProvider; +import org.sleuthkit.autopsy.coreutils.Version; +import org.sleuthkit.autopsy.ingest.FileIngestModule; +import org.sleuthkit.autopsy.ingest.IngestModuleFactory; +import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter; +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; +import org.sleuthkit.autopsy.modules.yara.ui.YaraIngestSettingsPanel; + +/** + * A factory that creates ingest modules that use the Yara rule set definitions + * to identify files that may be of interest to the user. + */ +@ServiceProvider(service = IngestModuleFactory.class) +public class YaraIngestModuleFactory extends IngestModuleFactoryAdapter { + + @Messages({ + "Yara_Module_Name=YARA", + "Yara_Module_Description=With the YARA ingest module you use YARA rule files to search files for textual or binary patterns." + }) + + @Override + public String getModuleDisplayName() { + return getModuleName(); + } + + @Override + public String getModuleDescription() { + return Bundle.Yara_Module_Description(); + } + + @Override + public String getModuleVersionNumber() { + return Version.getVersion(); + } + + @Override + public boolean hasIngestJobSettingsPanel() { + return true; + } + + @Override + public IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings) { + return new YaraIngestSettingsPanel((YaraIngestJobSettings)settings); + } + + @Override + public IngestModuleIngestJobSettings getDefaultIngestJobSettings() { + return new YaraIngestJobSettings(new ArrayList<>(), true); + } + + @Override + public boolean isFileIngestModuleFactory() { + return true; + } + + @Override + public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings settings) { + return new YaraIngestModule((YaraIngestJobSettings) settings); + } + + /** + * Return the name of the ingest module. + * + * @return Ingest module name. + */ + static String getModuleName() { + return Bundle.Yara_Module_Name(); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/rules/RuleSet.java b/Core/src/org/sleuthkit/autopsy/modules/yara/rules/RuleSet.java new file mode 100755 index 0000000000..112cf9206c --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/rules/RuleSet.java @@ -0,0 +1,84 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.yara.rules; + +import java.io.File; +import java.io.Serializable; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; + +/** + * Represents a yara rule set which is a collection of yara rule files. + */ +public class RuleSet implements Comparable, Serializable { + + private static final long serialVersionUID = 1L; + + private final String name; + private final Path path; + + /** + * Construct a new RuleSet. + * + * @param name Name of the rule set. + * @param path Directory path to the rule set. + */ + RuleSet(String name, Path path) { + this.name = name; + this.path = path; + } + + /** + * Returns the name of the rule set. + * + * @return Name of rule set. + */ + public String getName() { + return name; + } + + /** + * Returns location if the rule set files. + * + * @return The path for this rule set. + */ + public Path getPath() { + return path; + } + + /** + * Returns a list of the rule files in this rule set. + * + * @return List of Files in current directory. + */ + public List getRuleFiles() { + return Arrays.asList(path.toFile().listFiles()); + } + + @Override + public String toString() { + return getName(); + } + + @Override + public int compareTo(RuleSet ruleSet) { + return getName().compareTo(ruleSet.getName()); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/rules/RuleSetException.java b/Core/src/org/sleuthkit/autopsy/modules/yara/rules/RuleSetException.java new file mode 100755 index 0000000000..7ae0e12c45 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/rules/RuleSetException.java @@ -0,0 +1,47 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.yara.rules; + +/** + * + * An exception class for yara rule sets. + */ +public class RuleSetException extends Exception { + + private static final long serialVersionUID = 1L; + + /** + * Construct a RuleSetException with the given message. + * + * @param msg Exception message. + */ + RuleSetException(String msg) { + super(msg); + } + + /** + * Construct a RuleSetException with the given message and exception. + * + * @param msg Exception message. + * @param ex Exception. + */ + RuleSetException(String msg, Exception ex) { + super(msg, ex); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/rules/RuleSetManager.java b/Core/src/org/sleuthkit/autopsy/modules/yara/rules/RuleSetManager.java new file mode 100755 index 0000000000..eab2f80819 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/rules/RuleSetManager.java @@ -0,0 +1,109 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.yara.rules; + +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import org.sleuthkit.autopsy.coreutils.PlatformUtil; + +/** + * + * Yara Rule Set Manager. Manages the creation, deletion of yara rule sets. + */ +public class RuleSetManager { + + private final static String BASE_FOLDER = "yara"; + private final static String RULE_SET_FOLDER = "ruleSets"; + + /** + * Create a new Yara rule set with the given set name. + * + * @param name Name of new rule set + * + * @return Newly created RuleSet + * + * @throws RuleSetException RuleSet with given name already exists. + */ + public RuleSet createRuleSet(String name) throws RuleSetException { + + if (isRuleSetExists(name)) { + throw new RuleSetException(String.format("Yara rule set with name %s already exits.", name)); + } + + Path basePath = getRuleSetPath(); + Path setPath = Paths.get(basePath.toString(), name); + + setPath.toFile().mkdir(); + + return new RuleSet(name, setPath); + } + + /** + * Returns a list of all of the existing yara rule sets. + * + * @return + */ + public List getRuleSetList() { + List ruleSets = new ArrayList<>(); + Path basePath = getRuleSetPath(); + + String[] ruleSetNames = basePath.toFile().list(); + + for (String setName : ruleSetNames) { + ruleSets.add(new RuleSet(setName, Paths.get(basePath.toString(), setName))); + } + + return ruleSets; + } + + /** + * Check if a yara rule set of the given name exists. + * + * @param name Yara rule set name + * + * @return True if the rule set exist. + */ + public boolean isRuleSetExists(String name) { + Path basePath = getRuleSetPath(); + Path setPath = Paths.get(basePath.toString(), name); + + return setPath.toFile().exists(); + } + + /** + * Returns the Path to get RuleSet directory. If it does not exist it is + * created. + * + * @return Yara rule set directory path. + */ + private Path getRuleSetPath() { + Path basePath = Paths.get(PlatformUtil.getUserDirectory().getAbsolutePath(), BASE_FOLDER, RULE_SET_FOLDER); + File baseFile = basePath.toFile(); + + if (!baseFile.exists()) { + baseFile.mkdirs(); + } + + return basePath; + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/ui/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/Bundle.properties new file mode 100755 index 0000000000..22f969f628 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/Bundle.properties @@ -0,0 +1,15 @@ +OptionCategory_Name_YaraRuleSetOption=Yara Rule Sets +OptionCategory_Keywords_YaraRuleSetOption=YaraRuleSets +RuleSetPanel.newButton.text=New Set +RuleSetPanel.deleteButton.text=Delete Set +RuleSetPanel.ruleSetListLabel.text=Rule Sets: +RuleSetDetailsPanel.ruleSetListLabel.text=The folder currently contains the following rule files: +RuleSetDetailsPanel.setDetailsLabel.text=Set Details +RuleSetDetailsPanel.openFolderButton.text=Open Folder +RuleSetPanel.descriptionField.text=This module allows you to find files the match Yara rules. Each set has a list of Yara rule files. A file need only match one rule in the set to be found. +RuleSetDetailsPanel.openLabel.text=Place rule files in the set's folder. They will be compiled before use. +YaraIngestSettingsPanel.border.title=Select YARA rule sets to enable during ingest: +YaraIngestSettingsPanel.allFilesButton.text=All Files +YaraIngestSettingsPanel.allFilesButton.toolTipText= +YaraIngestSettingsPanel.executableFilesButton.text=Only Executable Files +RuleSetDetailsPanel.refreshButton.text=Refresh File List diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/ui/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/Bundle.properties-MERGED new file mode 100755 index 0000000000..f152cf959b --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/Bundle.properties-MERGED @@ -0,0 +1,22 @@ +OptionCategory_Name_YaraRuleSetOption=Yara Rule Sets +OptionCategory_Keywords_YaraRuleSetOption=YaraRuleSets +RuleSetDetailsPanel_failed_to_open_folder_msg=Failed to open new window for rule set file. +RuleSetDetailsPanel_failed_to_open_folder_title=Open Error +RuleSetPanel.newButton.text=New Set +RuleSetPanel.deleteButton.text=Delete Set +RuleSetPanel.ruleSetListLabel.text=Rule Sets: +RuleSetDetailsPanel.ruleSetListLabel.text=The folder currently contains the following rule files: +RuleSetDetailsPanel.setDetailsLabel.text=Set Details +RuleSetDetailsPanel.openFolderButton.text=Open Folder +RuleSetPanel.descriptionField.text=This module allows you to find files the match Yara rules. Each set has a list of Yara rule files. A file need only match one rule in the set to be found. +RuleSetDetailsPanel.openLabel.text=Place rule files in the set's folder. They will be compiled before use. +YaraIngestSettingsPanel.border.title=Select YARA rule sets to enable during ingest: +YaraIngestSettingsPanel.allFilesButton.text=All Files +YaraIngestSettingsPanel.allFilesButton.toolTipText= +YaraIngestSettingsPanel.executableFilesButton.text=Only Executable Files +RuleSetDetailsPanel.refreshButton.text=Refresh File List +# {0} - rule set name +YaraRuleSetOptionPanel_badName_msg=Rule set name {0} already exists.\nRule set names must be unique. +YaraRuleSetOptionPanel_badName_title=Create Rule Set +YaraRuleSetOptionPanel_new_rule_set_name_msg=Supply a new unique rule set name: +YaraRuleSetOptionPanel_new_rule_set_name_title=Rule Set Name diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/ui/RuleSetDetailsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/RuleSetDetailsPanel.form new file mode 100755 index 0000000000..955199108a --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/RuleSetDetailsPanel.form @@ -0,0 +1,113 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/ui/RuleSetDetailsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/RuleSetDetailsPanel.java new file mode 100755 index 0000000000..7e3e506948 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/RuleSetDetailsPanel.java @@ -0,0 +1,228 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.yara.ui; + +import java.awt.Component; +import java.awt.Desktop; +import java.awt.Graphics; +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.swing.DefaultListCellRenderer; +import javax.swing.DefaultListModel; +import javax.swing.JList; +import javax.swing.JOptionPane; +import org.openide.util.NbBundle.Messages; +import org.sleuthkit.autopsy.modules.yara.rules.RuleSet; + +/** + * A panel for displaying the details of an individual yara rule set. + */ +public class RuleSetDetailsPanel extends javax.swing.JPanel { + + private static final long serialVersionUID = 1L; + + private static final Logger logger = Logger.getLogger(RuleSetDetailsPanel.class.getName()); + + private RuleSet currentRuleSet; + private final DefaultListModel fileListModel; + private final JList fileList; + + /** + * Creates new form RuleSetDetailsPanel + */ + public RuleSetDetailsPanel() { + initComponents(); + + fileListModel = new DefaultListModel<>(); + fileList = new JList<>(); + fileList.setModel(fileListModel); + fileList.setCellRenderer(new FileRenderer()); + openFolderButton.setEnabled(false); + scrollPane.setViewportView(fileList); + + } + + /** + * Update the panel to show the details of the given RuleSet. + * + * @param ruleSet New RuleSet to display + */ + void setRuleSet(RuleSet ruleSet) { + currentRuleSet = ruleSet; + + fileListModel.clear(); + + if (ruleSet != null) { + List files = currentRuleSet.getRuleFiles(); + + if(files != null) { + for (File file : files) { + fileListModel.addElement(file); + } + } + } + + openFolderButton.setEnabled(ruleSet != null); + } + + /** + * Simple ListCellRenderer for the file list. + */ + private final class FileRenderer extends DefaultListCellRenderer { + + private static final long serialVersionUID = 1L; + + @Override + public Component getListCellRendererComponent(JList list, Object value, int index, boolean isSelected, boolean cellHasFocus) { + super.getListCellRendererComponent(list, value, index, isSelected, cellHasFocus); + + if (value instanceof File) { + File file = (File) value; + setText(file.getName()); + setToolTipText(file.getAbsolutePath()); + } + + return this; + } + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. + */ + @SuppressWarnings("unchecked") + // //GEN-BEGIN:initComponents + private void initComponents() { + java.awt.GridBagConstraints gridBagConstraints; + + javax.swing.JLabel ruleSetListLabel = new javax.swing.JLabel(); + javax.swing.JLabel setDetailsLabel = new javax.swing.JLabel(); + openFolderButton = new javax.swing.JButton(); + openLabel = new javax.swing.JLabel(); + scrollPane = new javax.swing.JScrollPane(); + javax.swing.JButton refreshButton = new javax.swing.JButton(); + + setLayout(new java.awt.GridBagLayout()); + + org.openide.awt.Mnemonics.setLocalizedText(ruleSetListLabel, org.openide.util.NbBundle.getMessage(RuleSetDetailsPanel.class, "RuleSetDetailsPanel.ruleSetListLabel.text")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 3; + gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST; + gridBagConstraints.insets = new java.awt.Insets(16, 0, 3, 0); + add(ruleSetListLabel, gridBagConstraints); + + org.openide.awt.Mnemonics.setLocalizedText(setDetailsLabel, org.openide.util.NbBundle.getMessage(RuleSetDetailsPanel.class, "RuleSetDetailsPanel.setDetailsLabel.text")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 0; + gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST; + gridBagConstraints.insets = new java.awt.Insets(0, 0, 10, 0); + add(setDetailsLabel, gridBagConstraints); + + openFolderButton.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/images/folder-icon-16.png"))); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(openFolderButton, org.openide.util.NbBundle.getMessage(RuleSetDetailsPanel.class, "RuleSetDetailsPanel.openFolderButton.text")); // NOI18N + openFolderButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + openFolderButtonActionPerformed(evt); + } + }); + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 1; + gridBagConstraints.gridy = 5; + gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST; + gridBagConstraints.insets = new java.awt.Insets(10, 0, 0, 0); + add(openFolderButton, gridBagConstraints); + + org.openide.awt.Mnemonics.setLocalizedText(openLabel, org.openide.util.NbBundle.getMessage(RuleSetDetailsPanel.class, "RuleSetDetailsPanel.openLabel.text")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 5; + gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST; + gridBagConstraints.insets = new java.awt.Insets(10, 0, 0, 0); + add(openLabel, gridBagConstraints); + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 4; + gridBagConstraints.gridwidth = 3; + gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH; + gridBagConstraints.weightx = 1.0; + gridBagConstraints.weighty = 1.0; + gridBagConstraints.insets = new java.awt.Insets(5, 0, 0, 0); + add(scrollPane, gridBagConstraints); + + refreshButton.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/images/arrow-circle-double-135.png"))); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(refreshButton, org.openide.util.NbBundle.getMessage(RuleSetDetailsPanel.class, "RuleSetDetailsPanel.refreshButton.text")); // NOI18N + refreshButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + refreshButtonActionPerformed(evt); + } + }); + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 2; + gridBagConstraints.gridy = 5; + gridBagConstraints.anchor = java.awt.GridBagConstraints.SOUTHEAST; + add(refreshButton, gridBagConstraints); + }// //GEN-END:initComponents + + @Messages({ + "RuleSetDetailsPanel_failed_to_open_folder_msg=Failed to open new window for rule set file.", + "RuleSetDetailsPanel_failed_to_open_folder_title=Open Error" + }) + private void openFolderButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_openFolderButtonActionPerformed + if (currentRuleSet != null) { + File file = currentRuleSet.getPath().toFile(); + if (file.exists()) { + try { + Desktop.getDesktop().open(file); + } catch (IOException ex) { + JOptionPane.showMessageDialog(this, + Bundle.RuleSetDetailsPanel_failed_to_open_folder_msg(), + Bundle.RuleSetDetailsPanel_failed_to_open_folder_title(), + JOptionPane.ERROR_MESSAGE); + logger.log(Level.WARNING, String.format("Failed to open external file explorer for: %s", currentRuleSet.getPath().toString()), ex); + } + } + } + }//GEN-LAST:event_openFolderButtonActionPerformed + + private void refreshButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_refreshButtonActionPerformed + if (currentRuleSet != null) { + fileListModel.clear(); + List files = currentRuleSet.getRuleFiles(); + + if(files != null) { + for (File file : files) { + fileListModel.addElement(file); + } + } + } + }//GEN-LAST:event_refreshButtonActionPerformed + + + // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JButton openFolderButton; + private javax.swing.JLabel openLabel; + private javax.swing.JScrollPane scrollPane; + // End of variables declaration//GEN-END:variables +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/ui/RuleSetPanel.form b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/RuleSetPanel.form new file mode 100755 index 0000000000..bd3e12390f --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/RuleSetPanel.form @@ -0,0 +1,110 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/ui/RuleSetPanel.java b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/RuleSetPanel.java new file mode 100755 index 0000000000..63474a8adb --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/RuleSetPanel.java @@ -0,0 +1,231 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.yara.ui; + +import java.awt.Component; +import java.awt.event.ActionListener; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import javax.swing.DefaultListCellRenderer; +import javax.swing.DefaultListModel; +import javax.swing.JList; +import javax.swing.ListSelectionModel; +import javax.swing.event.ListSelectionEvent; +import javax.swing.event.ListSelectionListener; +import org.sleuthkit.autopsy.modules.yara.rules.RuleSet; + +/** + * Panel for managing yara rule sets. + */ +public final class RuleSetPanel extends javax.swing.JPanel { + + private static final long serialVersionUID = 1L; + + private final DefaultListModel listModel; + private final JList ruleSetList; + + public RuleSetPanel() { + initComponents(); + + // Creating and initializing JList here to better take + // advantace of JList use of generics. + ruleSetList = new JList<>(); + listModel = new DefaultListModel<>(); + scrollPane.setViewportView(ruleSetList); + ruleSetList.setModel(listModel); + ruleSetList.setSelectionMode(ListSelectionModel.SINGLE_SELECTION); + ruleSetList.setCellRenderer(new RuleSetRenderer()); + ruleSetList.addListSelectionListener(new ListSelectionListener() { + @Override + public void valueChanged(ListSelectionEvent e) { + deleteButton.setEnabled(getSelectedRule() != null); + } + }); + + deleteButton.setEnabled(false); + } + + /** + * Add the list of rule sets to the list model. + * + * @param ruleSetList Rule sets to add. + */ + void addSetList(List newSetList) { + // Put the list into alphectical order. + List list = new ArrayList<>(); + list.addAll(newSetList); + Collections.sort(list); + + listModel.clear(); + + for (RuleSet set : list) { + listModel.addElement(set); + } + } + + /** + * Adds a new rule to the list. + * + * @param set + */ + void addRuleSet(RuleSet set) { + // This will assure that the new item + // appears in the correct location. + List list = Collections.list(listModel.elements()); + list.add(set); + + addSetList(list); + + ruleSetList.setSelectedValue(set, true); + } + + /** + * Removes a rule set from the list. + * + * @param set + */ + void removeRuleSet(RuleSet set) { + listModel.removeElement(set); + } + + /** + * Add a listener for the new rule set action. + * + * @param listener + */ + void addNewRuleListener(ActionListener listener) { + newButton.addActionListener(listener); + } + + /** + * Add a listener for the delete rule set action. + * + * @param listener + */ + void addDeleteRuleListener(ActionListener listener) { + deleteButton.addActionListener(listener); + } + + /** + * Add a listener for list selection change. + * + * @param listener + */ + void addListSelectionListener(ListSelectionListener listener) { + ruleSetList.addListSelectionListener(listener); + } + + /** + * Returns the current selected rule set. + * + * @return Currently selected rule set or null if one is not selected. + */ + RuleSet getSelectedRule() { + return ruleSetList.getSelectedValue(); + } + + /** + * Simple ListCellRenderer for a RuleSet. + */ + private final class RuleSetRenderer extends DefaultListCellRenderer { + + private static final long serialVersionUID = 1L; + + @Override + public Component getListCellRendererComponent(JList list, Object value, int index, boolean isSelected, boolean cellHasFocus) { + super.getListCellRendererComponent(list, value, index, isSelected, cellHasFocus); + + if (value instanceof RuleSet) { + RuleSet set = (RuleSet) value; + setText(set.getName()); + setToolTipText(set.getName()); + } + + return this; + } + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. + */ + @SuppressWarnings("unchecked") + // //GEN-BEGIN:initComponents + private void initComponents() { + java.awt.GridBagConstraints gridBagConstraints; + + javax.swing.JTextPane descriptionField = new javax.swing.JTextPane(); + javax.swing.JLabel ruleSetListLabel = new javax.swing.JLabel(); + scrollPane = new javax.swing.JScrollPane(); + javax.swing.JPanel buttonPanel = new javax.swing.JPanel(); + newButton = new javax.swing.JButton(); + deleteButton = new javax.swing.JButton(); + + setLayout(new java.awt.GridBagLayout()); + + descriptionField.setEditable(false); + descriptionField.setBorder(javax.swing.BorderFactory.createLineBorder(new java.awt.Color(0, 0, 0))); + descriptionField.setText(org.openide.util.NbBundle.getMessage(RuleSetPanel.class, "RuleSetPanel.descriptionField.text")); // NOI18N + descriptionField.setOpaque(false); + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL; + gridBagConstraints.weightx = 1.0; + add(descriptionField, gridBagConstraints); + + org.openide.awt.Mnemonics.setLocalizedText(ruleSetListLabel, org.openide.util.NbBundle.getMessage(RuleSetPanel.class, "RuleSetPanel.ruleSetListLabel.text")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 1; + gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST; + gridBagConstraints.insets = new java.awt.Insets(15, 0, 0, 0); + add(ruleSetListLabel, gridBagConstraints); + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 2; + gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH; + gridBagConstraints.weightx = 1.0; + gridBagConstraints.weighty = 1.0; + gridBagConstraints.insets = new java.awt.Insets(5, 0, 10, 0); + add(scrollPane, gridBagConstraints); + + buttonPanel.setLayout(new java.awt.GridLayout(1, 0, 5, 0)); + + newButton.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/images/add16.png"))); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(newButton, org.openide.util.NbBundle.getMessage(RuleSetPanel.class, "RuleSetPanel.newButton.text")); // NOI18N + buttonPanel.add(newButton); + + deleteButton.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/images/delete16.png"))); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(deleteButton, org.openide.util.NbBundle.getMessage(RuleSetPanel.class, "RuleSetPanel.deleteButton.text")); // NOI18N + buttonPanel.add(deleteButton); + + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST; + add(buttonPanel, gridBagConstraints); + }// //GEN-END:initComponents + + + // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JButton deleteButton; + private javax.swing.JButton newButton; + private javax.swing.JScrollPane scrollPane; + // End of variables declaration//GEN-END:variables +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraIngestSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraIngestSettingsPanel.form new file mode 100755 index 0000000000..d4832c66a1 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraIngestSettingsPanel.form @@ -0,0 +1,87 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraIngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraIngestSettingsPanel.java new file mode 100755 index 0000000000..e68b11ccb2 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraIngestSettingsPanel.java @@ -0,0 +1,182 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.yara.ui; + +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import javax.swing.DefaultListModel; +import org.sleuthkit.autopsy.guicomponeontutils.AbstractCheckboxListItem; +import org.sleuthkit.autopsy.guiutils.CheckBoxJList; +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; +import org.sleuthkit.autopsy.modules.yara.YaraIngestJobSettings; +import org.sleuthkit.autopsy.modules.yara.rules.RuleSet; +import org.sleuthkit.autopsy.modules.yara.rules.RuleSetManager; + +/** + * Yara Ingest settings panel. + */ +public class YaraIngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { + + private static final long serialVersionUID = 1L; + + private final CheckBoxJList checkboxList; + private final DefaultListModel listModel; + + /** + * Creates new form YaraIngestSettingsPanel + */ + YaraIngestSettingsPanel() { + initComponents(); + listModel = new DefaultListModel<>(); + checkboxList = new CheckBoxJList<>(); + scrollPane.setViewportView(checkboxList); + } + + public YaraIngestSettingsPanel(YaraIngestJobSettings settings) { + this(); + + List setNames = settings.getSelectedRuleSetNames(); + + checkboxList.setModel(listModel); + checkboxList.setOpaque(false); + RuleSetManager manager = new RuleSetManager(); + List ruleSetList = manager.getRuleSetList(); + for (RuleSet set : ruleSetList) { + RuleSetListItem item = new RuleSetListItem(set); + item.setChecked(setNames.contains(set.getName())); + listModel.addElement(item); + } + + allFilesButton.setSelected(!settings.onlyExecutableFiles()); + executableFilesButton.setSelected(settings.onlyExecutableFiles()); + } + + @Override + public IngestModuleIngestJobSettings getSettings() { + List selectedRules = new ArrayList<>(); + + Enumeration enumeration = listModel.elements(); + while (enumeration.hasMoreElements()) { + RuleSetListItem item = enumeration.nextElement(); + if (item.isChecked()) { + selectedRules.add(item.getRuleSet()); + } + } + + return new YaraIngestJobSettings(selectedRules, executableFilesButton.isSelected()); + } + + /** + * RuleSet wrapper class for Checkbox JList model. + */ + private final class RuleSetListItem extends AbstractCheckboxListItem { + + private final RuleSet ruleSet; + + /** + * RuleSetListItem constructor. + * + * @param set RuleSet object to display in list. + */ + RuleSetListItem(RuleSet set) { + this.ruleSet = set; + } + + /** + * Returns the RuleSet. + * + * @return + */ + RuleSet getRuleSet() { + return ruleSet; + } + + @Override + public String getDisplayName() { + return ruleSet.getName(); + } + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. + */ + @SuppressWarnings("unchecked") + // //GEN-BEGIN:initComponents + private void initComponents() { + java.awt.GridBagConstraints gridBagConstraints; + + buttonGroup = new javax.swing.ButtonGroup(); + scrollPane = new javax.swing.JScrollPane(); + buttonPanel = new javax.swing.JPanel(); + allFilesButton = new javax.swing.JRadioButton(); + executableFilesButton = new javax.swing.JRadioButton(); + + setBorder(javax.swing.BorderFactory.createTitledBorder(org.openide.util.NbBundle.getMessage(YaraIngestSettingsPanel.class, "YaraIngestSettingsPanel.border.title"))); // NOI18N + setLayout(new java.awt.GridBagLayout()); + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 0; + gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.weightx = 1.0; + gridBagConstraints.weighty = 1.0; + gridBagConstraints.insets = new java.awt.Insets(0, 5, 5, 5); + add(scrollPane, gridBagConstraints); + + buttonPanel.setLayout(new java.awt.GridBagLayout()); + + buttonGroup.add(allFilesButton); + org.openide.awt.Mnemonics.setLocalizedText(allFilesButton, org.openide.util.NbBundle.getMessage(YaraIngestSettingsPanel.class, "YaraIngestSettingsPanel.allFilesButton.text")); // NOI18N + allFilesButton.setToolTipText(org.openide.util.NbBundle.getMessage(YaraIngestSettingsPanel.class, "YaraIngestSettingsPanel.allFilesButton.toolTipText")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 0; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + buttonPanel.add(allFilesButton, gridBagConstraints); + + buttonGroup.add(executableFilesButton); + executableFilesButton.setSelected(true); + org.openide.awt.Mnemonics.setLocalizedText(executableFilesButton, org.openide.util.NbBundle.getMessage(YaraIngestSettingsPanel.class, "YaraIngestSettingsPanel.executableFilesButton.text")); // NOI18N + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 1; + gridBagConstraints.gridy = 0; + gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + buttonPanel.add(executableFilesButton, gridBagConstraints); + + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 1; + gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST; + gridBagConstraints.weightx = 1.0; + add(buttonPanel, gridBagConstraints); + }// //GEN-END:initComponents + + + // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JRadioButton allFilesButton; + private javax.swing.ButtonGroup buttonGroup; + private javax.swing.JPanel buttonPanel; + private javax.swing.JRadioButton executableFilesButton; + private javax.swing.JScrollPane scrollPane; + // End of variables declaration//GEN-END:variables +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraRuleSetOptionPanel.form b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraRuleSetOptionPanel.form new file mode 100755 index 0000000000..c7b05ede35 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraRuleSetOptionPanel.form @@ -0,0 +1,104 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraRuleSetOptionPanel.java b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraRuleSetOptionPanel.java new file mode 100755 index 0000000000..937dc504a3 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraRuleSetOptionPanel.java @@ -0,0 +1,207 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.yara.ui; + +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.io.File; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.swing.JOptionPane; +import javax.swing.event.ListSelectionEvent; +import javax.swing.event.ListSelectionListener; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.modules.yara.rules.RuleSet; +import org.sleuthkit.autopsy.modules.yara.rules.RuleSetException; +import org.sleuthkit.autopsy.modules.yara.rules.RuleSetManager; + +/** + * + * Yara Rule Set Option Panel. + */ +public class YaraRuleSetOptionPanel extends javax.swing.JPanel { + + private static final long serialVersionUID = 1L; + + private static final Logger logger = Logger.getLogger(YaraRuleSetOptionPanel.class.getName()); + + private final RuleSetManager manager; + + /** + * Creates new form YaraRuleSetOptionPanel + */ + public YaraRuleSetOptionPanel() { + initComponents(); + + manager = new RuleSetManager(); + + ruleSetPanel.addListSelectionListener(new ListSelectionListener() { + @Override + public void valueChanged(ListSelectionEvent e) { + handleSelectionChange(); + } + }); + + ruleSetPanel.addDeleteRuleListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + handleDeleteRuleSet(); + } + }); + + ruleSetPanel.addNewRuleListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + handleNewRuleSet(); + } + }); + } + + /** + * Update the panel with the current rule set. + */ + void updatePanel() { + ruleSetPanel.addSetList(manager.getRuleSetList()); + } + + /** + * Handle the change in rule set selection. Update the detail panel with the + * selected rule. + */ + private void handleSelectionChange() { + ruleSetDetailsPanel.setRuleSet(ruleSetPanel.getSelectedRule()); + } + + @NbBundle.Messages({ + "YaraRuleSetOptionPanel_new_rule_set_name_msg=Supply a new unique rule set name:", + "YaraRuleSetOptionPanel_new_rule_set_name_title=Rule Set Name", + "# {0} - rule set name", + "YaraRuleSetOptionPanel_badName_msg=Rule set name {0} already exists.\nRule set names must be unique.", + "YaraRuleSetOptionPanel_badName_title=Create Rule Set" + }) + /** + * Handle the new rule set action. Prompt the user for a rule set name, + * create the new set and update the rule set list. + */ + private void handleNewRuleSet() { + String value = JOptionPane.showInputDialog(this, + Bundle.YaraRuleSetOptionPanel_new_rule_set_name_msg(), + Bundle.YaraRuleSetOptionPanel_new_rule_set_name_title()); + try { + ruleSetPanel.addRuleSet(manager.createRuleSet(value)); + } catch (RuleSetException ex) { + JOptionPane.showMessageDialog(this, + Bundle.YaraRuleSetOptionPanel_badName_msg(value), + Bundle.YaraRuleSetOptionPanel_badName_title(), + JOptionPane.ERROR_MESSAGE); + logger.log(Level.WARNING, "Failed to create new rule set, user provide existing name.", ex); + } + } + + /** + * Handle the delete rule action. Delete the rule set and update the the + * rule set list. + */ + private void handleDeleteRuleSet() { + RuleSet ruleSet = ruleSetPanel.getSelectedRule(); + ruleSetPanel.removeRuleSet(ruleSet); + deleteDirectory(ruleSet.getPath().toFile()); + } + + /** + * Recursively delete the given directory and its children. + * + * @param directoryToBeDeleted + * + * @return True if the delete was successful. + */ + private boolean deleteDirectory(File directoryToBeDeleted) { + File[] allContents = directoryToBeDeleted.listFiles(); + if (allContents != null) { + for (File file : allContents) { + deleteDirectory(file); + } + } + return directoryToBeDeleted.delete(); + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. + */ + @SuppressWarnings("unchecked") + // //GEN-BEGIN:initComponents + private void initComponents() { + java.awt.GridBagConstraints gridBagConstraints; + + javax.swing.JScrollPane scrollPane = new javax.swing.JScrollPane(); + javax.swing.JPanel viewportPanel = new javax.swing.JPanel(); + javax.swing.JSeparator separator = new javax.swing.JSeparator(); + ruleSetDetailsPanel = new org.sleuthkit.autopsy.modules.yara.ui.RuleSetDetailsPanel(); + ruleSetPanel = new org.sleuthkit.autopsy.modules.yara.ui.RuleSetPanel(); + + setBorder(javax.swing.BorderFactory.createLineBorder(new java.awt.Color(0, 0, 0))); + setLayout(new java.awt.BorderLayout()); + + scrollPane.setBorder(null); + + viewportPanel.setLayout(new java.awt.GridBagLayout()); + + separator.setOrientation(javax.swing.SwingConstants.VERTICAL); + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 1; + gridBagConstraints.fill = java.awt.GridBagConstraints.VERTICAL; + gridBagConstraints.weighty = 1.0; + viewportPanel.add(separator, gridBagConstraints); + + ruleSetDetailsPanel.setMinimumSize(new java.awt.Dimension(500, 97)); + ruleSetDetailsPanel.setPreferredSize(new java.awt.Dimension(500, 204)); + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 2; + gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH; + gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST; + gridBagConstraints.weightx = 1.0; + gridBagConstraints.weighty = 1.0; + gridBagConstraints.insets = new java.awt.Insets(10, 10, 10, 10); + viewportPanel.add(ruleSetDetailsPanel, gridBagConstraints); + + ruleSetPanel.setMaximumSize(new java.awt.Dimension(400, 2147483647)); + ruleSetPanel.setMinimumSize(new java.awt.Dimension(400, 107)); + ruleSetPanel.setPreferredSize(new java.awt.Dimension(400, 214)); + gridBagConstraints = new java.awt.GridBagConstraints(); + gridBagConstraints.gridx = 0; + gridBagConstraints.gridy = 0; + gridBagConstraints.fill = java.awt.GridBagConstraints.VERTICAL; + gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST; + gridBagConstraints.weighty = 1.0; + gridBagConstraints.insets = new java.awt.Insets(10, 10, 10, 10); + viewportPanel.add(ruleSetPanel, gridBagConstraints); + + scrollPane.setViewportView(viewportPanel); + + add(scrollPane, java.awt.BorderLayout.CENTER); + }// //GEN-END:initComponents + + + // Variables declaration - do not modify//GEN-BEGIN:variables + private org.sleuthkit.autopsy.modules.yara.ui.RuleSetDetailsPanel ruleSetDetailsPanel; + private org.sleuthkit.autopsy.modules.yara.ui.RuleSetPanel ruleSetPanel; + // End of variables declaration//GEN-END:variables +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraRuleSetsOptionPanelController.java b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraRuleSetsOptionPanelController.java new file mode 100755 index 0000000000..7619e23b4c --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraRuleSetsOptionPanelController.java @@ -0,0 +1,90 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.yara.ui; + +import java.beans.PropertyChangeListener; +import javax.swing.JComponent; +import org.netbeans.spi.options.OptionsPanelController; +import org.openide.util.HelpCtx; +import org.openide.util.Lookup; + +/** + * Yara rule set option panel controller. + * + */ +@OptionsPanelController.TopLevelRegistration( + categoryName = "#OptionCategory_Name_YaraRuleSetOption", + iconBase = "org/sleuthkit/autopsy/images/yara_32.png", + keywords = "#OptionCategory_Keywords_YaraRuleSetOption", + keywordsCategory = "YaraRuleSets", + position = 20 +) +public class YaraRuleSetsOptionPanelController extends OptionsPanelController { + + private YaraRuleSetOptionPanel panel = null; + + @Override + public void update() { + if (panel != null) { + panel.updatePanel(); + } + } + + @Override + public void applyChanges() { + + } + + @Override + public void cancel() { + + } + + @Override + public JComponent getComponent(Lookup masterLookup) { + panel = new YaraRuleSetOptionPanel(); + return panel; + } + + @Override + public HelpCtx getHelpCtx() { + return null; + } + + @Override + public void addPropertyChangeListener(PropertyChangeListener pl) { + + } + + @Override + public void removePropertyChangeListener(PropertyChangeListener pl) { + + } + + @Override + public boolean isValid() { + return true; + } + + @Override + public boolean isChanged() { + return false; + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java b/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java index 67b253a1fa..7221ee4a55 100644 --- a/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java +++ b/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java @@ -60,6 +60,7 @@ import org.sleuthkit.autopsy.coreutils.FileTypeUtils.FileTypeCategory; import org.sleuthkit.autopsy.report.ReportProgressPanel; import org.sleuthkit.caseuco.CaseUcoExporter; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Blackboard.BlackboardException; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifactTag; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -983,10 +984,10 @@ public class PortableCaseReportModule implements ReportModule { BlackboardArtifact.Type oldCustomType = currentCase.getSleuthkitCase().getArtifactType(oldArtifact.getArtifactTypeName()); try { - BlackboardArtifact.Type newCustomType = portableSkCase.addBlackboardArtifactType(oldCustomType.getTypeName(), oldCustomType.getDisplayName()); + BlackboardArtifact.Type newCustomType = portableSkCase.getBlackboard().getOrAddArtifactType(oldCustomType.getTypeName(), oldCustomType.getDisplayName()); oldArtTypeIdToNewArtTypeId.put(oldArtifact.getArtifactTypeID(), newCustomType.getTypeID()); return newCustomType.getTypeID(); - } catch (TskDataException ex) { + } catch (BlackboardException ex) { throw new TskCoreException("Error creating new artifact type " + oldCustomType.getTypeName(), ex); // NON-NLS } } @@ -1007,11 +1008,11 @@ public class PortableCaseReportModule implements ReportModule { } try { - BlackboardAttribute.Type newCustomType = portableSkCase.addArtifactAttributeType(oldAttrType.getTypeName(), + BlackboardAttribute.Type newCustomType = portableSkCase.getBlackboard().getOrAddAttributeType(oldAttrType.getTypeName(), oldAttrType.getValueType(), oldAttrType.getDisplayName()); oldAttrTypeIdToNewAttrType.put(oldAttribute.getAttributeType().getTypeID(), newCustomType); return newCustomType; - } catch (TskDataException ex) { + } catch (BlackboardException ex) { throw new TskCoreException("Error creating new attribute type " + oldAttrType.getTypeName(), ex); // NON-NLS } } diff --git a/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/RecentFilesSummaryTest.java b/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/RecentFilesSummaryTest.java index 068f0c3904..ff73c428b9 100644 --- a/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/RecentFilesSummaryTest.java +++ b/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/RecentFilesSummaryTest.java @@ -71,8 +71,8 @@ public class RecentFilesSummaryTest { * Means of acquiring data from a method in RecentFilesSummary. * * @param recentFilesSummary The RecentFilesSummary object. - * @param dataSource The datasource. - * @param count The number of items to retrieve. + * @param dataSource The datasource. + * @param count The number of items to retrieve. * * @return The method's return data. * @@ -95,7 +95,7 @@ public class RecentFilesSummaryTest { /** * If -1 count passed to method, should throw IllegalArgumentException. * - * @param method The method to call. + * @param method The method to call. * @param methodName The name of the metho * * @throws TskCoreException @@ -137,7 +137,7 @@ public class RecentFilesSummaryTest { * SleuthkitCase isn't called. * * @param recentFilesMethod The method to call. - * @param methodName The name of the method + * @param methodName The name of the method * * @throws SleuthkitCaseProviderException * @throws TskCoreException @@ -175,7 +175,7 @@ public class RecentFilesSummaryTest { * If SleuthkitCase returns no results, an empty list is returned. * * @param recentFilesMethod The method to call. - * @param methodName The name of the method. + * @param methodName The name of the method. * * @throws SleuthkitCaseProviderException * @throws TskCoreException @@ -220,11 +220,11 @@ public class RecentFilesSummaryTest { /** * Gets a mock BlackboardArtifact. * - * @param ds The data source to which the artifact belongs. - * @param artifactId The artifact id. - * @param artType The artifact type. + * @param ds The data source to which the artifact belongs. + * @param artifactId The artifact id. + * @param artType The artifact type. * @param attributeArgs The mapping of attribute type to value for each - * attribute in the artifact. + * attribute in the artifact. * * @return The mock artifact. */ @@ -247,10 +247,10 @@ public class RecentFilesSummaryTest { /** * Returns a mock artifact for getRecentlyOpenedDocuments. * - * @param ds The datasource for the artifact. + * @param ds The datasource for the artifact. * @param artifactId The artifact id. - * @param dateTime The time in seconds from epoch. - * @param path The path for the document. + * @param dateTime The time in seconds from epoch. + * @param path The path for the document. * * @return The mock artifact with pertinent attributes. */ @@ -292,13 +292,33 @@ public class RecentFilesSummaryTest { } } + @Test + public void getRecentlyOpenedDocuments_uniquePaths() throws SleuthkitCaseProviderException, TskCoreException { + DataSource dataSource = TskMockUtils.getDataSource(1); + + BlackboardArtifact item1 = getRecentDocumentArtifact(dataSource, 1001, DAY_SECONDS, "/a/path"); + BlackboardArtifact item2 = getRecentDocumentArtifact(dataSource, 1002, DAY_SECONDS + 1, "/a/path"); + BlackboardArtifact item3 = getRecentDocumentArtifact(dataSource, 1003, DAY_SECONDS + 2, "/a/path"); + List artifacts = Arrays.asList(item2, item3, item1); + + Pair casePair = DataSourceSummaryMockUtils.getArtifactsTSKMock(RandomizationUtils.getMixedUp(artifacts)); + RecentFilesSummary summary = new RecentFilesSummary(() -> casePair.getLeft()); + List results = summary.getRecentlyOpenedDocuments(dataSource, 10); + + // verify results (only successItem) + Assert.assertNotNull(results); + Assert.assertEquals(1, results.size()); + Assert.assertEquals((Long) (DAY_SECONDS + 2), results.get(0).getDateAsLong()); + Assert.assertTrue("/a/path".equalsIgnoreCase(results.get(0).getPath())); + } + @Test public void getRecentlyOpenedDocuments_filtersMissingData() throws SleuthkitCaseProviderException, TskCoreException { DataSource dataSource = TskMockUtils.getDataSource(1); BlackboardArtifact successItem = getRecentDocumentArtifact(dataSource, 1001, DAY_SECONDS, "/a/path"); BlackboardArtifact nullTime = getRecentDocumentArtifact(dataSource, 1002, null, "/a/path2"); - BlackboardArtifact zeroTime = getRecentDocumentArtifact(dataSource, 10021, 0L, "/a/path2a"); + BlackboardArtifact zeroTime = getRecentDocumentArtifact(dataSource, 10021, 0L, "/a/path2a"); List artifacts = Arrays.asList(nullTime, zeroTime, successItem); Pair casePair = DataSourceSummaryMockUtils.getArtifactsTSKMock(RandomizationUtils.getMixedUp(artifacts)); @@ -315,11 +335,11 @@ public class RecentFilesSummaryTest { /** * Creates a mock blackboard artifact for getRecentDownloads. * - * @param ds The datasource. + * @param ds The datasource. * @param artifactId The artifact id. - * @param dateTime The time in seconds from epoch. - * @param domain The domain. - * @param path The path for the download. + * @param dateTime The time in seconds from epoch. + * @param domain The domain. + * @param path The path for the download. * * @return The mock artifact. */ @@ -368,6 +388,30 @@ public class RecentFilesSummaryTest { } } + @Test + public void getRecentDownloads_uniquePaths() throws SleuthkitCaseProviderException, TskCoreException { + DataSource dataSource = TskMockUtils.getDataSource(1); + + BlackboardArtifact item1 = getRecentDownloadArtifact(dataSource, 1001, DAY_SECONDS, "domain1.com", "/a/path1"); + BlackboardArtifact item1a = getRecentDownloadArtifact(dataSource, 10011, DAY_SECONDS + 1, "domain1.com", "/a/path1"); + BlackboardArtifact item2 = getRecentDownloadArtifact(dataSource, 1002, DAY_SECONDS + 2, "domain2.com", "/a/path1"); + BlackboardArtifact item3 = getRecentDownloadArtifact(dataSource, 1003, DAY_SECONDS + 3, "domain2a.com", "/a/path1"); + List artifacts = Arrays.asList(item2, item3, item1); + + Pair casePair = DataSourceSummaryMockUtils.getArtifactsTSKMock(RandomizationUtils.getMixedUp(artifacts)); + RecentFilesSummary summary = new RecentFilesSummary(() -> casePair.getLeft()); + + // call method + List results = summary.getRecentDownloads(dataSource, 10); + + // verify results + Assert.assertNotNull(results); + Assert.assertEquals(1, results.size()); + Assert.assertEquals((Long) (DAY_SECONDS + 3), results.get(0).getDateAsLong()); + Assert.assertTrue("/a/path1".equalsIgnoreCase(results.get(0).getPath())); + Assert.assertTrue("domain2a.com".equalsIgnoreCase(results.get(0).getWebDomain())); + } + @Test public void getRecentDownloads_filtersMissingData() throws SleuthkitCaseProviderException, TskCoreException { DataSource dataSource = TskMockUtils.getDataSource(1); @@ -409,19 +453,17 @@ public class RecentFilesSummaryTest { * Constructor with all parameters. * * @param messageArtifactTypeId The type id for the artifact or null if - * no message artifact to be created. - * @param emailFrom Who the message is from or null not to - * include attribute. - * @param messageTime Time in seconds from epoch or null not - * to include attribute. - * @param fileParentPath The parent AbstractFile's path value. - * @param fileName The parent AbstractFile's filename - * value. - * @param associatedAttrFormed If false, the TSK_ASSOCIATED_OBJECT - * artifact has no attribute (even though - * it is required). - * @param hasParent Whether or not the artifact has a parent - * AbstractFile. + * no message artifact to be created. + * @param emailFrom Who the message is from or null not to include + * attribute. + * @param messageTime Time in seconds from epoch or null not to include + * attribute. + * @param fileParentPath The parent AbstractFile's path value. + * @param fileName The parent AbstractFile's filename value. + * @param associatedAttrFormed If false, the TSK_ASSOCIATED_OBJECT + * artifact has no attribute (even though it is required). + * @param hasParent Whether or not the artifact has a parent + * AbstractFile. */ AttachmentArtifactItem(Integer messageArtifactTypeId, String emailFrom, Long messageTime, String fileParentPath, String fileName, @@ -441,14 +483,13 @@ public class RecentFilesSummaryTest { * SleuthkitCase assumed. * * @param messageArtifactTypeId The type id for the artifact or null if - * no message artifact to be created. - * @param emailFrom Who the message is from or null not to - * include attribute. - * @param messageTime Time in seconds from epoch or null not - * to include attribute. - * @param fileParentPath The parent AbstractFile's path value. - * @param fileName The parent AbstractFile's filename - * value. + * no message artifact to be created. + * @param emailFrom Who the message is from or null not to include + * attribute. + * @param messageTime Time in seconds from epoch or null not to include + * attribute. + * @param fileParentPath The parent AbstractFile's path value. + * @param fileName The parent AbstractFile's filename value. */ AttachmentArtifactItem(Integer messageArtifactTypeId, String emailFrom, Long messageTime, String fileParentPath, String fileName) { this(messageArtifactTypeId, emailFrom, messageTime, fileParentPath, fileName, true, true); @@ -486,11 +527,11 @@ public class RecentFilesSummaryTest { /** * Sets up the associated artifact message for the TSK_ASSOCIATED_OBJECT. * - * @param artifacts The mapping of artifact id to artifact. - * @param item The record to setup. - * @param dataSource The datasource. + * @param artifacts The mapping of artifact id to artifact. + * @param item The record to setup. + * @param dataSource The datasource. * @param associatedId The associated attribute id. - * @param artifactId The artifact id. + * @param artifactId The artifact id. * * @return The associated Artifact blackboard attribute. * @@ -504,7 +545,7 @@ public class RecentFilesSummaryTest { if (item.getMessageArtifactTypeId() == null) { return associatedAttr; } - + // find the artifact type or null if not found ARTIFACT_TYPE messageType = Stream.of(ARTIFACT_TYPE.values()) .filter((artType) -> artType.getTypeID() == item.getMessageArtifactTypeId()) @@ -534,7 +575,7 @@ public class RecentFilesSummaryTest { * to return pertinent data. * * @param items Each attachment item where each item could represent a - * return result if fully formed. + * return result if fully formed. * * @return The mock SleuthkitCase and Blackboard. */ @@ -678,4 +719,34 @@ public class RecentFilesSummaryTest { .toString().equalsIgnoreCase(successItem2Details.getPath())); Assert.assertTrue(successItem2.getEmailFrom().equalsIgnoreCase(successItem2Details.getSender())); } + + @Test + public void getRecentAttachments_uniquePath() throws SleuthkitCaseProviderException, TskCoreException { + // setup data + DataSource dataSource = TskMockUtils.getDataSource(1); + + AttachmentArtifactItem item1 = new AttachmentArtifactItem(ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID(), + "person@sleuthkit.com", DAY_SECONDS, "/parent/path", "msg.pdf"); + AttachmentArtifactItem item2 = new AttachmentArtifactItem(ARTIFACT_TYPE.TSK_MESSAGE.getTypeID(), + "person_on_skype", DAY_SECONDS + 1, "/parent/path", "msg.pdf"); + AttachmentArtifactItem item3 = new AttachmentArtifactItem(ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID(), + "person2@sleuthkit.com", DAY_SECONDS + 2, "/parent/path", "msg.pdf"); + + List items = Arrays.asList(item1, item2, item3); + + Pair casePair = getRecentAttachmentArtifactCase(items); + RecentFilesSummary summary = new RecentFilesSummary(() -> casePair.getLeft()); + + // get data + List results = summary.getRecentAttachments(dataSource, 10); + + // verify results + Assert.assertNotNull(results); + Assert.assertEquals(1, results.size()); + + Assert.assertEquals(results.get(0).getDateAsLong(), (Long) (DAY_SECONDS + 2)); + Assert.assertTrue(Paths.get(item3.getFileParentPath(), item3.getFileName()) + .toString().equalsIgnoreCase(results.get(0).getPath())); + Assert.assertTrue(results.get(0).getSender().equalsIgnoreCase(item3.getEmailFrom())); + } } diff --git a/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/UserActivitySummaryTest.java b/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/UserActivitySummaryTest.java index 0aff8a56b5..c7f3246a4d 100644 --- a/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/UserActivitySummaryTest.java +++ b/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/UserActivitySummaryTest.java @@ -66,18 +66,22 @@ import org.sleuthkit.datamodel.TskCoreException; * Tests for UserActivitySummary. */ public class UserActivitySummaryTest { + /** - * Function to retrieve data from UserActivitySummary with the provided arguments. + * Function to retrieve data from UserActivitySummary with the provided + * arguments. */ private interface DataFunction { + /** * A UserActivitySummary method encapsulated in a uniform manner. + * * @param userActivitySummary The UserActivitySummary class to use. * @param datasource The data source. * @param count The count. * @return The list of objects to return. * @throws SleuthkitCaseProviderException - * @throws TskCoreException + * @throws TskCoreException */ List retrieve(UserActivitySummary userActivitySummary, DataSource datasource, int count) throws SleuthkitCaseProviderException, TskCoreException; @@ -117,8 +121,8 @@ public class UserActivitySummaryTest { /** * Gets a UserActivitySummary class to test. * - * @param tskCase The SleuthkitCase. - * @param hasTranslation Whether the translation service is functional. + * @param tskCase The SleuthkitCase. + * @param hasTranslation Whether the translation service is functional. * @param translateFunction Function for translation. * * @return The UserActivitySummary class to use for testing. @@ -333,6 +337,28 @@ public class UserActivitySummaryTest { } } + @Test + public void getRecentDevices_uniqueByDeviceId() + throws TskCoreException, NoServiceProviderException, SleuthkitCaseProviderException, TskCoreException, TranslationException { + + long dataSourceId = 1L; + DataSource dataSource = TskMockUtils.getDataSource(dataSourceId); + BlackboardArtifact item1 = getRecentDeviceArtifact(1001, dataSource, "ID1", "MAKE1", "MODEL1", DAY_SECONDS); + BlackboardArtifact item2 = getRecentDeviceArtifact(1002, dataSource, "ID1", "MAKE1", "MODEL1", DAY_SECONDS + 1); + BlackboardArtifact item3 = getRecentDeviceArtifact(1003, dataSource, "ID1", "MAKE1", "MODEL1", DAY_SECONDS + 2); + + Pair tskPair = getArtifactsTSKMock(Arrays.asList(item1, item2, item3)); + UserActivitySummary summary = getTestClass(tskPair.getLeft(), false, null); + + List results = summary.getRecentDevices(dataSource, 10); + + Assert.assertEquals(1, results.size()); + Assert.assertEquals((long) (DAY_SECONDS + 2), results.get(0).getDateAccessed().getTime() / 1000); + Assert.assertTrue("ID1".equalsIgnoreCase(results.get(0).getDeviceId())); + Assert.assertTrue("MAKE1".equalsIgnoreCase(results.get(0).getDeviceMake())); + Assert.assertTrue("MODEL1".equalsIgnoreCase(results.get(0).getDeviceModel())); + } + private static BlackboardArtifact getWebSearchArtifact(long artifactId, DataSource dataSource, String query, Long date) { try { return TskMockUtils.getArtifact(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_WEB_SEARCH_QUERY), artifactId, dataSource, @@ -708,8 +734,8 @@ public class UserActivitySummaryTest { * * @param artifactId The artifact id. * @param dataSource The datasource. - * @param dateRcvd The date received in seconds or null to exclude. - * @param dateSent The date sent in seconds or null to exclude. + * @param dateRcvd The date received in seconds or null to exclude. + * @param dateSent The date sent in seconds or null to exclude. * * @return The mock artifact. */ @@ -738,8 +764,8 @@ public class UserActivitySummaryTest { * * @param artifactId The artifact id. * @param dataSource The datasource. - * @param dateStart The date start in seconds or null to exclude. - * @param dateEnd The date end in seconds or null to exclude. + * @param dateStart The date start in seconds or null to exclude. + * @param dateEnd The date end in seconds or null to exclude. * * @return The mock artifact. */ @@ -768,8 +794,8 @@ public class UserActivitySummaryTest { * * @param artifactId The artifact id. * @param dataSource The datasource. - * @param type The account type. - * @param dateSent The date of the message in seconds. + * @param type The account type. + * @param dateSent The date of the message in seconds. */ private static BlackboardArtifact getMessageArtifact(long artifactId, DataSource dataSource, String type, Long dateTime) { List attributes = new ArrayList<>(); @@ -794,11 +820,11 @@ public class UserActivitySummaryTest { /** * Performs a test on UserActivitySummary.getRecentAccounts. * - * @param dataSource The datasource to use as parameter. - * @param count The count to use as a parameter. - * @param retArtifacts The artifacts to return from - * SleuthkitCase.getArtifacts. This method filters - * based on artifact type from the call. + * @param dataSource The datasource to use as parameter. + * @param count The count to use as a parameter. + * @param retArtifacts The artifacts to return from + * SleuthkitCase.getArtifacts. This method filters based on artifact type + * from the call. * @param expectedResults The expected results. * * @throws TskCoreException diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties-MERGED b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties-MERGED index 56a675e256..823399e0d0 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties-MERGED +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties-MERGED @@ -205,7 +205,9 @@ DeleteCaseTask.progress.parsingManifest=Parsing manifest file {0}... DeleteCaseTask.progress.releasingManifestLock=Releasing lock on the manifest file {0}... DeleteCaseTask.progress.startMessage=Starting deletion... DeleteOrphanCaseNodesAction.progressDisplayName=Cleanup Case Znodes +# {0} - item count DeleteOrphanCaseNodesDialog.additionalInit.lblNodeCount.text=Znodes found: {0} +# {0} - item count DeleteOrphanCaseNodesDialog.additionalInit.znodesTextArea.countMessage=ZNODES FOUND: {0} DeleteOrphanCaseNodesTask.progress.connectingToCoordSvc=Connecting to the coordination service # {0} - node path diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/SharedConfiguration.java b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/SharedConfiguration.java index fbbbd1999b..f7c9c6d8ff 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/SharedConfiguration.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/SharedConfiguration.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2015 Basis Technology Corp. + * Copyright 2015 - 2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -151,6 +151,8 @@ public class SharedConfiguration { /** * Upload the current multi-user ingest settings to a shared folder. * + * @return + * * @throws SharedConfigurationException * @throws CoordinationServiceException * @throws InterruptedException @@ -208,6 +210,7 @@ public class SharedConfiguration { uploadCentralRepositorySettings(remoteFolder); uploadObjectDetectionClassifiers(remoteFolder); uploadPythonModules(remoteFolder); + uploadYARASetting(remoteFolder); try { Files.deleteIfExists(uploadInProgress.toPath()); @@ -222,6 +225,8 @@ public class SharedConfiguration { /** * Download the multi-user settings from a shared folder. * + * @return + * * @throws SharedConfigurationException * @throws InterruptedException */ @@ -252,13 +257,16 @@ public class SharedConfiguration { } try { - /* Make sure all recent changes are saved to the preference file. - This also releases open file handles to the preference files. If this - is not done, then occasionally downloading of shared configuration - fails silently, likely because Java/OS is still holding the file handle. - The problem manifests itself by some of the old/original configuration files - sticking around after shared configuration has seemingly been successfully - updated. */ + /* + * Make sure all recent changes are saved to the preference + * file. This also releases open file handles to the preference + * files. If this is not done, then occasionally downloading of + * shared configuration fails silently, likely because Java/OS + * is still holding the file handle. The problem manifests + * itself by some of the old/original configuration files + * sticking around after shared configuration has seemingly been + * successfully updated. + */ UserPreferences.saveToStorage(); } catch (BackingStoreException ex) { throw new SharedConfigurationException("Failed to save shared configuration settings", ex); @@ -275,6 +283,7 @@ public class SharedConfiguration { downloadCentralRepositorySettings(remoteFolder); downloadObjectDetectionClassifiers(remoteFolder); downloadPythonModules(remoteFolder); + downloadYARASettings(remoteFolder); // Download general settings, then restore the current // values for the unshared fields @@ -344,7 +353,7 @@ public class SharedConfiguration { private void saveNonSharedSettings() { sharedConfigMaster = AutoIngestUserPreferences.getSharedConfigMaster(); sharedConfigFolder = AutoIngestUserPreferences.getSharedConfigFolder(); - showToolsWarning = AutoIngestUserPreferences.getShowToolsWarning(); + showToolsWarning = AutoIngestUserPreferences.getShowToolsWarning(); displayLocalTime = UserPreferences.displayTimesInLocalTime(); hideKnownFilesInDataSource = UserPreferences.hideKnownFilesInDataSourcesTree(); hideKnownFilesInViews = UserPreferences.hideKnownFilesInViewsTree(); @@ -360,7 +369,7 @@ public class SharedConfiguration { private void restoreNonSharedSettings() { AutoIngestUserPreferences.setSharedConfigFolder(sharedConfigFolder); AutoIngestUserPreferences.setSharedConfigMaster(sharedConfigMaster); - AutoIngestUserPreferences.setShowToolsWarning(showToolsWarning); + AutoIngestUserPreferences.setShowToolsWarning(showToolsWarning); UserPreferences.setDisplayTimesInLocalTime(displayLocalTime); UserPreferences.setHideKnownFilesInDataSourcesTree(hideKnownFilesInDataSource); UserPreferences.setHideKnownFilesInViewsTree(hideKnownFilesInViews); @@ -515,21 +524,23 @@ public class SharedConfiguration { throw new SharedConfigurationException(String.format("Failed to copy %s to %s", remoteFile.getAbsolutePath(), localSettingsFolder.getAbsolutePath()), ex); } } - + /** - * Copy an entire local settings folder to the remote folder, deleting any existing files. - * + * Copy an entire local settings folder to the remote folder, deleting any + * existing files. + * * @param localFolder The local folder to copy - * @param remoteBaseFolder The remote folder that will hold a copy of the original folder - * - * @throws SharedConfigurationException + * @param remoteBaseFolder The remote folder that will hold a copy of the + * original folder + * + * @throws SharedConfigurationException */ private void copyLocalFolderToRemoteFolder(File localFolder, File remoteBaseFolder) throws SharedConfigurationException { logger.log(Level.INFO, "Uploading {0} to {1}", new Object[]{localFolder.getAbsolutePath(), remoteBaseFolder.getAbsolutePath()}); - + File newRemoteFolder = new File(remoteBaseFolder, localFolder.getName()); - - if(newRemoteFolder.exists()) { + + if (newRemoteFolder.exists()) { try { FileUtils.deleteDirectory(newRemoteFolder); } catch (IOException ex) { @@ -537,29 +548,30 @@ public class SharedConfiguration { throw new SharedConfigurationException(String.format("Failed to delete remote folder {0}", newRemoteFolder.getAbsolutePath()), ex); } } - + try { FileUtils.copyDirectoryToDirectory(localFolder, remoteBaseFolder); } catch (IOException ex) { throw new SharedConfigurationException(String.format("Failed to copy %s to %s", localFolder, remoteBaseFolder.getAbsolutePath()), ex); - } + } } - + /** - * Copy an entire remote settings folder to the local folder, deleting any existing files. - * No error if the remote folder does not exist. - * + * Copy an entire remote settings folder to the local folder, deleting any + * existing files. No error if the remote folder does not exist. + * * @param localFolder The local folder that will be overwritten. - * @param remoteBaseFolder The remote folder holding the folder that will be copied - * - * @throws SharedConfigurationException + * @param remoteBaseFolder The remote folder holding the folder that will be + * copied + * + * @throws SharedConfigurationException */ private void copyRemoteFolderToLocalFolder(File localFolder, File remoteBaseFolder) throws SharedConfigurationException { logger.log(Level.INFO, "Downloading {0} from {1}", new Object[]{localFolder.getAbsolutePath(), remoteBaseFolder.getAbsolutePath()}); - + // Clean out the local folder regardless of whether the remote version exists. leave the // folder in place since Autopsy expects it to exist. - if(localFolder.exists()) { + if (localFolder.exists()) { try { FileUtils.cleanDirectory(localFolder); } catch (IOException ex) { @@ -567,19 +579,19 @@ public class SharedConfiguration { throw new SharedConfigurationException(String.format("Failed to delete files from local folder {0}", localFolder.getAbsolutePath()), ex); } } - + File remoteSubFolder = new File(remoteBaseFolder, localFolder.getName()); - if(! remoteSubFolder.exists()) { + if (!remoteSubFolder.exists()) { logger.log(Level.INFO, "{0} does not exist", remoteSubFolder.getAbsolutePath()); return; } - + try { FileUtils.copyDirectory(remoteSubFolder, localFolder); } catch (IOException ex) { throw new SharedConfigurationException(String.format("Failed to copy %s from %s", localFolder, remoteBaseFolder.getAbsolutePath()), ex); - } - } + } + } /** * Upload the basic set of auto-ingest settings to the shared folder. @@ -899,56 +911,56 @@ public class SharedConfiguration { /** * Upload the object detection classifiers. - * + * * @param remoteFolder Shared settings folder - * - * @throws SharedConfigurationException + * + * @throws SharedConfigurationException */ private void uploadObjectDetectionClassifiers(File remoteFolder) throws SharedConfigurationException { publishTask("Uploading object detection classfiers"); File classifiersFolder = new File(PlatformUtil.getObjectDetectionClassifierPath()); copyLocalFolderToRemoteFolder(classifiersFolder, remoteFolder); } - + /** * Download the object detection classifiers. - * + * * @param remoteFolder Shared settings folder - * - * @throws SharedConfigurationException + * + * @throws SharedConfigurationException */ private void downloadObjectDetectionClassifiers(File remoteFolder) throws SharedConfigurationException { publishTask("Downloading object detection classfiers"); File classifiersFolder = new File(PlatformUtil.getObjectDetectionClassifierPath()); copyRemoteFolderToLocalFolder(classifiersFolder, remoteFolder); } - - /** + + /** * Upload the Python modules. - * + * * @param remoteFolder Shared settings folder - * - * @throws SharedConfigurationException + * + * @throws SharedConfigurationException */ private void uploadPythonModules(File remoteFolder) throws SharedConfigurationException { publishTask("Uploading python modules"); File classifiersFolder = new File(PlatformUtil.getUserPythonModulesPath()); copyLocalFolderToRemoteFolder(classifiersFolder, remoteFolder); } - + /** * Download the Python modules. - * + * * @param remoteFolder Shared settings folder - * - * @throws SharedConfigurationException + * + * @throws SharedConfigurationException */ private void downloadPythonModules(File remoteFolder) throws SharedConfigurationException { publishTask("Downloading python modules"); File classifiersFolder = new File(PlatformUtil.getUserPythonModulesPath()); copyRemoteFolderToLocalFolder(classifiersFolder, remoteFolder); } - + /** * Upload settings and hash databases to the shared folder. The general * algorithm is: - Copy the general settings in hashsets.xml - For each hash @@ -1093,12 +1105,10 @@ public class SharedConfiguration { Map remoteVersions = readVersionsFromFile(remoteVersionFile); /* - Iterate through remote list - If local needs it, download - - Download remote settings files to local - Download remote versions file to local - HashDbManager reload + * Iterate through remote list If local needs it, download + * + * Download remote settings files to local Download remote versions file + * to local HashDbManager reload */ File localDb = new File(""); File sharedDb = new File(""); @@ -1247,7 +1257,7 @@ public class SharedConfiguration { if (hashDb.getIndexPath().isEmpty() && hashDb.getDatabasePath().isEmpty()) { continue; } - + if (hashDb.hasIndexOnly()) { results.add(hashDb.getIndexPath()); } else { @@ -1356,4 +1366,41 @@ public class SharedConfiguration { throw new SharedConfigurationException(String.format("Failed to calculate CRC for %s", file.getAbsolutePath()), ex); } } + + /** + * Copy the YARA settings directory from the local directory to the remote + * directory. + * + * @param remoteFolder Shared settings folder + * + * @throws + * org.sleuthkit.autopsy.experimental.configuration.SharedConfiguration.SharedConfigurationException + */ + private void uploadYARASetting(File remoteFolder) throws SharedConfigurationException { + publishTask("Uploading YARA module configuration"); + + File localYara = Paths.get(PlatformUtil.getUserDirectory().getAbsolutePath(), "yara").toFile(); + + if (!localYara.exists()) { + return; + } + + copyLocalFolderToRemoteFolder(localYara, remoteFolder); + } + + /** + * Downloads the YARA settings folder from the remote directory to the local + * one. + * + * @param remoteFolder Shared settings folder + * + * @throws + * org.sleuthkit.autopsy.experimental.configuration.SharedConfiguration.SharedConfigurationException + */ + private void downloadYARASettings(File remoteFolder) throws SharedConfigurationException { + publishTask("Downloading YARA module configuration"); + File localYara = Paths.get(PlatformUtil.getUserDirectory().getAbsolutePath(), "yara").toFile(); + + copyRemoteFolderToLocalFolder(localYara, remoteFolder); + } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED index a842d76f74..7927d43415 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED @@ -61,6 +61,7 @@ ExtractOS_progressMessage=Checking for OS ExtractPrefetch_errMsg_prefetchParsingFailed={0}: Error analyzing prefetch files ExtractPrefetch_module_name=Windows Prefetch Extractor ExtractRecycleBin_module_name=Recycle Bin +ExtractRecycleBin_Recyle_Bin_Display_Name=Recycle Bin ExtractSafari_Error_Getting_History=An error occurred while processing Safari history files. ExtractSafari_Error_Parsing_Bookmark=An error occured while processing Safari Bookmark files ExtractSafari_Error_Parsing_Cookies=An error occured while processing Safari Cookies files @@ -84,16 +85,9 @@ ExtractZone_progress_Msg=Extracting :Zone.Identifer files ExtractZone_Restricted=Restricted Sites Zone ExtractZone_Trusted=Trusted Sites Zone OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Recent Activity ingest module.\n\nThe module extracts useful information about the recent user activity on the disk image being ingested, such as:\n\n- Recently open documents,\n- Web activity (sites visited, stored cookies, book marked sites, search engine queries, file downloads),\n- Recently attached devices,\n- Installed programs.\n\nThe module currently supports Windows only disk images.\nThe plugin is also fully functional when deployed on Windows version of Autopsy. +OpenIDE-Module-Long-Description=Recent Activity ingest module.\n\n\The module extracts useful information about the recent user activity on the disk image being ingested, such as:\n\n- Recently open documents,\n- Web activity (sites visited, stored cookies, book marked sites, search engine queries, file downloads),\n- Recently attached devices,\n- Installed programs.\n\nThe module currently supports Windows only disk images.\nThe plugin is also fully functional when deployed on Windows version of Autopsy. OpenIDE-Module-Name=RecentActivity OpenIDE-Module-Short-Description=Recent Activity finder ingest module -Browser.name.Microsoft.Edge=Microsoft Edge -Browser.name.Yandex=Yandex -Browser.name.Opera=Opera -Browser.name.SalamWeb=SalamWeb -Browser.name.UC.Browser=UC Browser -Browser.name.Brave=Brave -Browser.name.Google.Chrome=Google Chrome Chrome.moduleName=Chromium Chrome.getHistory.errMsg.errGettingFiles=Error when trying to get Chrome history files. Chrome.getHistory.errMsg.couldntFindAnyFiles=Could not find any allocated Chrome history files. diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java index adc7170482..175a47044e 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRecycleBin.java @@ -45,6 +45,7 @@ import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Blackboard.BlackboardException; import org.sleuthkit.datamodel.BlackboardArtifact; import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_OS_ACCOUNT; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -415,6 +416,9 @@ final class ExtractRecycleBin extends Extract { return artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.fromID(type.getTypeID()))); } + @Messages({ + "ExtractRecycleBin_Recyle_Bin_Display_Name=Recycle Bin" + }) /** * Create TSK_RECYCLE_BIN artifact type. * @@ -422,9 +426,9 @@ final class ExtractRecycleBin extends Extract { */ private void createRecycleBinArtifactType() throws TskCoreException { try { - tskCase.addBlackboardArtifactType(RECYCLE_BIN_ARTIFACT_NAME, "Recycle Bin"); //NON-NLS - } catch (TskDataException ex) { - logger.log(Level.INFO, String.format("%s may have already been defined for this case", RECYCLE_BIN_ARTIFACT_NAME)); + tskCase.getBlackboard().getOrAddArtifactType(RECYCLE_BIN_ARTIFACT_NAME, Bundle.ExtractRecycleBin_Recyle_Bin_Display_Name()); //NON-NLS + } catch (BlackboardException ex) { + throw new TskCoreException(String.format("An exception was thrown while defining artifact type %s", RECYCLE_BIN_ARTIFACT_NAME), ex); } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index 9d232cdc84..13ede70c90 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -68,6 +68,7 @@ import java.util.Set; import java.util.HashSet; import static java.util.Locale.US; import static java.util.TimeZone.getTimeZone; +import org.openide.util.Exceptions; import org.openide.util.Lookup; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; @@ -77,6 +78,7 @@ import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService; import org.sleuthkit.autopsy.recentactivity.ShellBagParser.ShellBag; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Account; +import org.sleuthkit.datamodel.Blackboard.BlackboardException; import org.sleuthkit.datamodel.BlackboardArtifact; import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_ASSOCIATED_OBJECT; import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_OS_ACCOUNT; @@ -1961,18 +1963,11 @@ class ExtractRegistry extends Extract { */ private BlackboardArtifact.Type getShellBagArtifact() throws TskCoreException { if (shellBagArtifactType == null) { - shellBagArtifactType = tskCase.getArtifactType(SHELLBAG_ARTIFACT_NAME); - - if (shellBagArtifactType == null) { - try { - tskCase.addBlackboardArtifactType(SHELLBAG_ARTIFACT_NAME, Bundle.Shellbag_Artifact_Display_Name()); //NON-NLS - } catch (TskDataException ex) { - // Artifact already exists - logger.log(Level.INFO, String.format("%s may have already been defined for this case", SHELLBAG_ARTIFACT_NAME)); - } - - shellBagArtifactType = tskCase.getArtifactType(SHELLBAG_ARTIFACT_NAME); - } + try { + shellBagArtifactType = tskCase.getBlackboard().getOrAddArtifactType(SHELLBAG_ARTIFACT_NAME, Bundle.Shellbag_Artifact_Display_Name()); + } catch (BlackboardException ex) { + throw new TskCoreException(String.format("Failed to get shell bag artifact type", SHELLBAG_ARTIFACT_NAME), ex); + } } return shellBagArtifactType; @@ -1989,12 +1984,12 @@ class ExtractRegistry extends Extract { private BlackboardAttribute.Type getLastWriteAttribute() throws TskCoreException { if (shellBagLastWriteAttributeType == null) { try { - shellBagLastWriteAttributeType = tskCase.addArtifactAttributeType(SHELLBAG_ATTRIBUTE_LAST_WRITE, + shellBagLastWriteAttributeType = tskCase.getBlackboard().getOrAddAttributeType(SHELLBAG_ATTRIBUTE_LAST_WRITE, BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.DATETIME, Bundle.Shellbag_Last_Write_Attribute_Display_Name()); - } catch (TskDataException ex) { + } catch (BlackboardException ex) { // Attribute already exists get it from the case - shellBagLastWriteAttributeType = tskCase.getAttributeType(SHELLBAG_ATTRIBUTE_LAST_WRITE); + throw new TskCoreException(String.format("Failed to get custom attribute %s", SHELLBAG_ATTRIBUTE_LAST_WRITE), ex); } } return shellBagLastWriteAttributeType; @@ -2011,12 +2006,11 @@ class ExtractRegistry extends Extract { private BlackboardAttribute.Type getKeyAttribute() throws TskCoreException { if (shellBagKeyAttributeType == null) { try { - shellBagKeyAttributeType = tskCase.addArtifactAttributeType(SHELLBAG_ATTRIBUTE_KEY, + shellBagKeyAttributeType = tskCase.getBlackboard().getOrAddAttributeType(SHELLBAG_ATTRIBUTE_KEY, BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, Bundle.Shellbag_Key_Attribute_Display_Name()); - } catch (TskDataException ex) { - // The attribute already exists get it from the case - shellBagKeyAttributeType = tskCase.getAttributeType(SHELLBAG_ATTRIBUTE_KEY); + } catch (BlackboardException ex) { + throw new TskCoreException(String.format("Failed to get key attribute %s", SHELLBAG_ATTRIBUTE_KEY), ex); } } return shellBagKeyAttributeType; diff --git a/Testing/build.xml b/Testing/build.xml index f0ba6531a4..f8eff4b821 100644 --- a/Testing/build.xml +++ b/Testing/build.xml @@ -76,6 +76,10 @@ + + + + diff --git a/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java b/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java index 6f6e04d7bd..f2aa476590 100644 --- a/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java +++ b/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2018 Basis Technology Corp. + * Copyright 2011-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,6 +25,9 @@ import junit.framework.Test; import junit.framework.TestCase; import org.netbeans.jemmy.Timeouts; import org.netbeans.junit.NbModuleSuite; +import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoDbChoice; +import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoDbManager; +import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoException; /** * This test expects the following system properties to be set: img_path: The @@ -100,6 +103,21 @@ public class RegressionTest extends TestCase { public void setUp() { logger.info("######## " + AutopsyTestCases.getEscapedPath(System.getProperty("img_path")) + " #######"); Timeouts.setDefault("ComponentOperator.WaitComponentTimeout", 1000000); + + try { + if (Boolean.parseBoolean(System.getProperty("isMultiUser"))) { + // Set up a custom postgres CR using the configuration passed + // to system properties. + CentralRepoDbManager manager = new CentralRepoDbManager(); + manager.getDbSettingsPostgres().setHost(System.getProperty("crHost")); + manager.getDbSettingsPostgres().setPort(Integer.parseInt(System.getProperty("crPort"))); + manager.getDbSettingsPostgres().setUserName(System.getProperty("crUserName")); + manager.getDbSettingsPostgres().setPassword(System.getProperty("crPassword")); + manager.setupPostgresDb(CentralRepoDbChoice.POSTGRESQL_CUSTOM); + } + } catch (CentralRepoException ex) { + throw new RuntimeException("Error setting up multi user CR", ex); + } } /** diff --git a/docs/doxygen-user/multi-user/installPostgres.dox b/docs/doxygen-user/multi-user/installPostgres.dox index fd7753281b..d56afe931b 100644 --- a/docs/doxygen-user/multi-user/installPostgres.dox +++ b/docs/doxygen-user/multi-user/installPostgres.dox @@ -13,7 +13,7 @@ You should ensure that the database folder is backed up. To install PostgreSQL, perform the following steps: -1. Download a 64-bit PostgreSQL installer from http://www.enterprisedb.com/products-services-training/pgdownload#windows Choose the one that says _Win X86-64_. Autopsy has been tested with PostgreSQL version 9.5. +1. Download a 64-bit PostgreSQL installer from https://www.enterprisedb.com/downloads/postgres-postgresql-downloads Choose one under Windows x86-64. Autopsy has been tested with PostgreSQL version 9.5. 2. Run the installer. The name will be similar to _postgresql-9.5.3-1-windows-x64.exe_. diff --git a/test/script/regression.py b/test/script/regression.py index 9999a5c563..94b50c244f 100644 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -471,6 +471,10 @@ class TestRunner(object): test_data.ant.append("-DsolrPort=" + str(test_config.solrPort)) test_data.ant.append("-DmessageServiceHost=" + test_config.messageServiceHost) test_data.ant.append("-DmessageServicePort=" + str(test_config.messageServicePort)) + test_data.ant.append("-DcrHost=" + str(test_config.crHost)) + test_data.ant.append("-DcrPort=" + str(test_config.crPort)) + test_data.ant.append("-DcrUserName=" + str(test_config.crUserName)) + test_data.ant.append("-DcrPassword=" + str(test_config.crPassword)) if test_data.isMultiUser: test_data.ant.append("-DisMultiUser=true") # Note: test_data has autopys_version attribute, but we couldn't see it from here. It's set after run ingest. @@ -854,6 +858,14 @@ class TestConfiguration(object): self.messageServicePort = parsed_config.getElementsByTagName("messageServicePort")[0].getAttribute("value").encode().decode("utf_8") if parsed_config.getElementsByTagName("multiUser_outdir"): self.multiUser_outdir = parsed_config.getElementsByTagName("multiUser_outdir")[0].getAttribute("value").encode().decode("utf_8") + if parsed_config.getElementsByTagName("crHost"): + self.crHost = parsed_config.getElementsByTagName("crHost")[0].getAttribute("value").encode().decode("utf_8") + if parsed_config.getElementsByTagName("crPort"): + self.crPort = parsed_config.getElementsByTagName("crPort")[0].getAttribute("value").encode().decode("utf_8") + if parsed_config.getElementsByTagName("crUserName"): + self.crUserName = parsed_config.getElementsByTagName("crUserName")[0].getAttribute("value").encode().decode("utf_8") + if parsed_config.getElementsByTagName("crPassword"): + self.crPassword = parsed_config.getElementsByTagName("crPassword")[0].getAttribute("value").encode().decode("utf_8") self._init_imgs(parsed_config) self._init_build_info(parsed_config) diff --git a/test/script/regression_utils.py b/test/script/regression_utils.py index 51fa3eb1c4..0c0229beb2 100644 --- a/test/script/regression_utils.py +++ b/test/script/regression_utils.py @@ -27,7 +27,7 @@ def make_os_path(platform, *dirs): path += str(dir).replace('\\', '/') + '/' return path_fix(path) elif platform == "win32": - return make_path(dirs) + return make_path(*dirs) else: print("Couldn't make path, because we only support Windows and Cygwin at this time.") sys.exit(1) diff --git a/thirdparty/iLeapp/ileapp.exe b/thirdparty/iLeapp/ileapp.exe index 8176b4f679..d17ab28f4d 100644 Binary files a/thirdparty/iLeapp/ileapp.exe and b/thirdparty/iLeapp/ileapp.exe differ diff --git a/thirdparty/yara/ReadMe.txt b/thirdparty/yara/ReadMe.txt index 31f38633b4..98c356f1b6 100755 --- a/thirdparty/yara/ReadMe.txt +++ b/thirdparty/yara/ReadMe.txt @@ -1,7 +1,7 @@ This folder contains the projects you need for building and testing the yarabridge.dll and YaraJNIWrapper.jar. bin: -Contains the built dll and jar. +Contains the built jar and jarac64.exe. jarac64.exe is used to by the ingest module to compile the rule files. yarabridge: VS project to create the dll that wraps the the libyara library. @@ -18,7 +18,8 @@ Steps for building yarabridge, YaraJNIWrapper and YaraWrapperTest. - Build Release x64. 3. Open the yarabridge project and build Release x64. -If you have link issues, make sure you build release x64 in the previous step. - -This project will automatically copy the built dll to the bin folder. + -This project will automatically copy the built dll into the YaraJNIWrapper src\org\sleuthkit\autopsy\yara folder. + - This is where is needs to be so that its included into the jar file. 4. Build YaraJNIWrapper - Open in netbeans and select Build. - Manually move the newly build jar file to the bin folder. After building the jar file can be found in diff --git a/thirdparty/yara/YaraJNIWrapper/nbproject/build-impl.xml b/thirdparty/yara/YaraJNIWrapper/nbproject/build-impl.xml index 38dd8d0c87..d5569a48c3 100755 --- a/thirdparty/yara/YaraJNIWrapper/nbproject/build-impl.xml +++ b/thirdparty/yara/YaraJNIWrapper/nbproject/build-impl.xml @@ -179,9 +179,7 @@ is divided into following sections: - - - + @@ -289,7 +287,6 @@ is divided into following sections: Must set src.dir - Must set test.src.dir Must set build.dir Must set dist.dir Must set build.classes.dir @@ -588,9 +585,6 @@ is divided into following sections: - - - @@ -613,11 +607,7 @@ is divided into following sections: - - - - - + @@ -1544,14 +1534,14 @@ is divided into following sections: - - + + - + @@ -1592,17 +1582,15 @@ is divided into following sections: - + - + - - - + @@ -1616,14 +1604,12 @@ is divided into following sections: Must select some files in the IDE or set javac.includes - + - - - + diff --git a/thirdparty/yara/YaraJNIWrapper/nbproject/private/private.xml b/thirdparty/yara/YaraJNIWrapper/nbproject/private/private.xml index 475096252c..f6db5d6149 100755 --- a/thirdparty/yara/YaraJNIWrapper/nbproject/private/private.xml +++ b/thirdparty/yara/YaraJNIWrapper/nbproject/private/private.xml @@ -1,4 +1,9 @@ + + + file:/C:/Users/kelly/Workspace/autopsy/thirdparty/yara/YaraJNIWrapper/src/org/sleuthkit/autopsy/yara/YaraJNIWrapper.java + + diff --git a/thirdparty/yara/YaraJNIWrapper/nbproject/project.properties b/thirdparty/yara/YaraJNIWrapper/nbproject/project.properties index a0ef4dac37..0af470a2bf 100755 --- a/thirdparty/yara/YaraJNIWrapper/nbproject/project.properties +++ b/thirdparty/yara/YaraJNIWrapper/nbproject/project.properties @@ -1,9 +1,10 @@ annotation.processing.enabled=true annotation.processing.enabled.in.editor=false -annotation.processing.processor.options= annotation.processing.processors.list= annotation.processing.run.all.processors=true annotation.processing.source.output=${build.generated.sources.dir}/ap-source-output +application.title=YaraJNIWrapper +application.vendor=kelly build.classes.dir=${build.dir}/classes build.classes.excludes=**/*.java,**/*.form # This directory is removed when the project is cleaned: @@ -32,10 +33,13 @@ dist.jar=${dist.dir}/YaraJNIWrapper.jar dist.javadoc.dir=${dist.dir}/javadoc dist.jlink.dir=${dist.dir}/jlink dist.jlink.output=${dist.jlink.dir}/YaraJNIWrapper +endorsed.classpath= excludes= +file.reference.yara-lib=src/org/sleuthkit/autopsy/yara/lib includes=** jar.compress=false -javac.classpath= +javac.classpath=\ + ${file.reference.yara-lib} # Space-separated list of extra javac options javac.compilerargs= javac.deprecation=false @@ -90,4 +94,3 @@ run.test.modulepath=\ ${javac.test.modulepath} source.encoding=UTF-8 src.dir=src -test.src.dir=test diff --git a/thirdparty/yara/YaraJNIWrapper/nbproject/project.xml b/thirdparty/yara/YaraJNIWrapper/nbproject/project.xml index df43138d7e..89ae97a48b 100755 --- a/thirdparty/yara/YaraJNIWrapper/nbproject/project.xml +++ b/thirdparty/yara/YaraJNIWrapper/nbproject/project.xml @@ -7,9 +7,7 @@ - - - + diff --git a/thirdparty/yara/YaraJNIWrapper/src/org/sleuthkit/autopsy/yara/YaraJNIWrapper.java b/thirdparty/yara/YaraJNIWrapper/src/org/sleuthkit/autopsy/yara/YaraJNIWrapper.java index 0fc5e8f0f4..b14cea7fd3 100755 --- a/thirdparty/yara/YaraJNIWrapper/src/org/sleuthkit/autopsy/yara/YaraJNIWrapper.java +++ b/thirdparty/yara/YaraJNIWrapper/src/org/sleuthkit/autopsy/yara/YaraJNIWrapper.java @@ -18,9 +18,11 @@ */ package org.sleuthkit.autopsy.yara; -import java.net.URISyntaxException; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -31,18 +33,12 @@ import java.util.logging.Logger; */ public class YaraJNIWrapper { - // Load the yarabridge.dll which should be located in the same directory as - // the jar file. If we need to use this code for debugging the dll this - // code will need to be modified to add that support. static { - Path directoryPath = null; try { - directoryPath = Paths.get(YaraJNIWrapper.class.getProtectionDomain().getCodeSource().getLocation().toURI()).getParent().toAbsolutePath(); - } catch (URISyntaxException ex) { + extractAndLoadDll(); + } catch (IOException | YaraWrapperException ex) { Logger.getLogger(YaraJNIWrapper.class.getName()).log(Level.SEVERE, null, ex); } - String libraryPath = Paths.get(directoryPath != null ? directoryPath.toString() : "", "yarabridge.dll").toAbsolutePath().toString(); - System.load(libraryPath); } /** @@ -50,19 +46,65 @@ public class YaraJNIWrapper { * * The rule path must be to a yara compile rule file. * - * @param compiledRulesPath - * @param byteBuffer + * @param compiledRulesPath Absolute path to a compiled YARA rule file. + * @param byteBuffer File buffer. + * @param bufferSize Size of the byte to read in the given buffer + * @param timeoutSec Scan timeout value in seconds. * * @return List of rules found rules. Null maybe returned if error occurred. * * @throws YaraWrapperException */ - static public native List findRuleMatch(String compiledRulesPath, byte[] byteBuffer) throws YaraWrapperException; + static public native List findRuleMatch(String compiledRulesPath, byte[] byteBuffer, int bufferSize, int timeoutSec) throws YaraWrapperException; + + /** + * Returns a list of matching YARA rules found in the given file. + * + * @param compiledRulePath Absolute path to a compiled YARA rule file. + * @param filePath Absolute path to the file to search. + * @param timeoutSec Scan timeout value in seconds. + * + * @return List of rules found rules. Null maybe returned if error occurred. + * + * + * @throws YaraWrapperException + */ + static public native List findRuleMatchFile(String compiledRulePath, String filePath, int timeoutSec) throws YaraWrapperException; + + /** + * Copy yarabridge.dll from inside the jar to a temp file that can be loaded + * with System.load. + * + * To make this work, the dll needs to be in the same folder as this source + * file. The dll needs to be located somewhere in the jar class path. + * + * @throws IOException + * @throws YaraWrapperException + */ + static private void extractAndLoadDll() throws IOException, YaraWrapperException { + File tempFile = File.createTempFile("lib", null); + tempFile.deleteOnExit(); + try (InputStream in = YaraJNIWrapper.class.getResourceAsStream("yarabridge.dll")) { + if (in == null) { + throw new YaraWrapperException("native library was not found in jar file."); + } + try (OutputStream out = new FileOutputStream(tempFile)) { + byte[] buffer = new byte[1024]; + int lengthRead; + while ((lengthRead = in.read(buffer)) > 0) { + out.write(buffer, 0, lengthRead); + out.flush(); + } + } + } + + System.load(tempFile.getAbsolutePath()); + } /** * private constructor. */ private YaraJNIWrapper() { } - + } diff --git a/thirdparty/yara/YaraJNIWrapper/src/org/sleuthkit/autopsy/yara/yarabridge.dll b/thirdparty/yara/YaraJNIWrapper/src/org/sleuthkit/autopsy/yara/yarabridge.dll new file mode 100755 index 0000000000..4be3a859e1 Binary files /dev/null and b/thirdparty/yara/YaraJNIWrapper/src/org/sleuthkit/autopsy/yara/yarabridge.dll differ diff --git a/thirdparty/yara/YaraWrapperTest/nbproject/project.properties b/thirdparty/yara/YaraWrapperTest/nbproject/project.properties index c0126ab42a..b7874aae82 100755 --- a/thirdparty/yara/YaraWrapperTest/nbproject/project.properties +++ b/thirdparty/yara/YaraWrapperTest/nbproject/project.properties @@ -35,7 +35,7 @@ dist.jlink.dir=${dist.dir}/jlink dist.jlink.output=${dist.jlink.dir}/YaraWrapperTest endorsed.classpath= excludes= -file.reference.YaraJNIWrapper.jar=../bin/YaraJNIWrapper.jar +file.reference.YaraJNIWrapper.jar=../YaraJNIWrapper/dist/YaraJNIWrapper.jar includes=** jar.compress=false javac.classpath=\ diff --git a/thirdparty/yara/YaraWrapperTest/src/org/sleuthkit/autopsy/yara/YaraWrapperTest.java b/thirdparty/yara/YaraWrapperTest/src/org/sleuthkit/autopsy/yara/YaraWrapperTest.java index 4a57abfef2..ee25676896 100755 --- a/thirdparty/yara/YaraWrapperTest/src/org/sleuthkit/autopsy/yara/YaraWrapperTest.java +++ b/thirdparty/yara/YaraWrapperTest/src/org/sleuthkit/autopsy/yara/YaraWrapperTest.java @@ -26,8 +26,6 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; -import org.sleuthkit.autopsy.yara.YaraJNIWrapper; -import org.sleuthkit.autopsy.yara.YaraWrapperException; /** * Tests the YaraJNIWrapper code. @@ -43,6 +41,7 @@ public class YaraWrapperTest { } testFileRuleMatch(args[0], args[1]); + testFileRuleMatchFile(args[0], args[1]); } /** @@ -58,7 +57,7 @@ public class YaraWrapperTest { try { byte[] data = Files.readAllBytes(path); - List list = YaraJNIWrapper.findRuleMatch(compiledRulePath, data); + List list = YaraJNIWrapper.findRuleMatch(compiledRulePath, data, data.length, 100); if (list != null) { if (list.isEmpty()) { @@ -77,5 +76,34 @@ public class YaraWrapperTest { logger.log(Level.SEVERE, "Error thrown from yarabridge", ex); } } + + /** + * Test the call to findRuleMatchFile which takes a compiled rule file + * path and a path to a file. + * + * @param compiledRulePath + * @param filePath + */ + private static void testFileRuleMatchFile(String compiledRulePath, String filePath) { + try { + List list = YaraJNIWrapper.findRuleMatchFile(compiledRulePath, filePath, 100); + + if (list != null) { + if (list.isEmpty()) { + System.out.println("FindRuleMatch return an empty list"); + } else { + System.out.println("Matching Rules:"); + for (String s : list) { + System.out.println(s); + } + } + } else { + logger.log(Level.SEVERE, "FindRuleMatch return a null list"); + } + + } catch (YaraWrapperException ex) { + logger.log(Level.SEVERE, "Error thrown from yarabridge", ex); + } + } } diff --git a/thirdparty/yara/bin/YaraJNIWrapper.jar b/thirdparty/yara/bin/YaraJNIWrapper.jar index 749d7a6ae7..21fe881a82 100755 Binary files a/thirdparty/yara/bin/YaraJNIWrapper.jar and b/thirdparty/yara/bin/YaraJNIWrapper.jar differ diff --git a/thirdparty/yara/bin/yarabridge.dll b/thirdparty/yara/bin/yarabridge.dll deleted file mode 100755 index c74062a626..0000000000 Binary files a/thirdparty/yara/bin/yarabridge.dll and /dev/null differ diff --git a/thirdparty/yara/bin/yarac64.exe b/thirdparty/yara/bin/yarac64.exe new file mode 100755 index 0000000000..bf94cc4462 Binary files /dev/null and b/thirdparty/yara/bin/yarac64.exe differ diff --git a/thirdparty/yara/yarabridge/yarabridge/YaraJNIWrapper.cpp b/thirdparty/yara/yarabridge/yarabridge/YaraJNIWrapper.cpp index 0d36d2a039..0fc7f63806 100755 --- a/thirdparty/yara/yarabridge/yarabridge/YaraJNIWrapper.cpp +++ b/thirdparty/yara/yarabridge/yarabridge/YaraJNIWrapper.cpp @@ -20,7 +20,6 @@ using std::string; using std::vector; - /* Callback method to be passed to yr_rules_scan_mem method. user_data is expected to be a pointer to a string vector. @@ -79,49 +78,85 @@ jobject createArrayList(JNIEnv *env, std::vector vector) { return list; } +/* + Loads the compiled rules file returning a YARA error code. + Throws a java exeception if there are any issues. +*/ +int loadRuleFile(JNIEnv * env, jstring compiledRulePath, YR_RULES **rules) { + char errorMessage[256]; + const char *nativeString = env->GetStringUTFChars(compiledRulePath, 0); + int result = yr_rules_load(nativeString, rules); + + if (result != ERROR_SUCCESS) { + sprintf_s(errorMessage, "Failed to load compiled yara rule %s (error code = %d)\n", nativeString, result); + throwException(env, errorMessage); + } + + env->ReleaseStringUTFChars(compiledRulePath, nativeString); + + return result; +} + +/* + Initalize the YARA library, if needed. yr_initialize only needs to be called once. +*/ +int initalizeYaraLibrary(JNIEnv * env) { + static int library_initalized = 0; + char errorMessage[256]; + int result = ERROR_SUCCESS; + if (library_initalized == 0) { + if ((result = yr_initialize()) != ERROR_SUCCESS) { + sprintf_s(errorMessage, "libyara initialization error (%d)\n", result); + throwException(env, errorMessage); + } + library_initalized = 1; + } + + return result; +} + /* * Class: org_sleuthkit_autopsy_yara_YaraJNIWrapper * Method: FindRuleMatch * Signature: (Ljava/lang/String;[B)Ljava/util/List; */ JNIEXPORT jobject JNICALL Java_org_sleuthkit_autopsy_yara_YaraJNIWrapper_findRuleMatch -(JNIEnv * env, jclass cls, jstring compiledRulePath, jbyteArray fileByteArray) { +(JNIEnv * env, jclass cls, jstring compiledRulePath, jbyteArray fileByteArray, jint byteArrayLength, jint timeoutSec) { char errorMessage[256]; - const char *nativeString = env->GetStringUTFChars(compiledRulePath, 0); jobject resultList = NULL; - int result; - if ((result = yr_initialize()) != ERROR_SUCCESS) { - sprintf_s(errorMessage, "libyara initialization error (%d)\n", result); - throwException(env, errorMessage); + YR_RULES *rules = NULL; + + if ((result = initalizeYaraLibrary(env)) != ERROR_SUCCESS) { return resultList; } + while (1) { - YR_RULES *rules = NULL; - if ((result = yr_rules_load(nativeString, &rules)) != ERROR_SUCCESS) { - sprintf_s(errorMessage, "Failed to load compiled yara rules (%d)\n", result); - throwException(env, errorMessage); + if((result = loadRuleFile(env, compiledRulePath, &rules)) != ERROR_SUCCESS) { break; } - boolean isCopy; - int byteArrayLength = env->GetArrayLength(fileByteArray); if (byteArrayLength == 0) { throwException(env, "Unable to scan for matches. File byte array size was 0."); break; } + boolean isCopy; jbyte* nativeByteArray = env->GetByteArrayElements(fileByteArray, &isCopy); - int flags = 0; std::vector scanResults; - result = yr_rules_scan_mem(rules, (unsigned char*)nativeByteArray, byteArrayLength, flags, callback, &scanResults, 1000000); + result = yr_rules_scan_mem(rules, (unsigned char*)nativeByteArray, byteArrayLength, 0, callback, &scanResults, timeoutSec); env->ReleaseByteArrayElements(fileByteArray, nativeByteArray, 0); if (result != ERROR_SUCCESS) { - sprintf_s(errorMessage, "Yara file scan failed (%d)\n", result); + if (result == ERROR_SCAN_TIMEOUT) { + sprintf_s(errorMessage, "Yara file scan timed out"); + } + else { + sprintf_s(errorMessage, "Yara file scan failed (%d)\n", result); + } throwException(env, errorMessage); break; } @@ -130,9 +165,60 @@ JNIEXPORT jobject JNICALL Java_org_sleuthkit_autopsy_yara_YaraJNIWrapper_findRul break; } - env->ReleaseStringUTFChars(compiledRulePath, nativeString); - yr_finalize(); + if (rules != NULL) { + yr_rules_destroy(rules); + } return resultList; +} + +/* +* Class: org_sleuthkit_autopsy_yara_YaraJNIWrapper +* Method: findRuleMatchFile +* Signature: (Ljava/lang/String;Ljava/lang/String;)Ljava/util/List; +*/ +JNIEXPORT jobject JNICALL Java_org_sleuthkit_autopsy_yara_YaraJNIWrapper_findRuleMatchFile +(JNIEnv * env, jclass cls, jstring compiledRulePath, jstring filePath, jint timeoutSec) { + + char errorMessage[256]; + jobject resultList = NULL; + int result; + YR_RULES *rules = NULL; + + if ((result = initalizeYaraLibrary(env)) != ERROR_SUCCESS) { + return resultList; + } + + + while (1) { + if ((result = loadRuleFile(env, compiledRulePath, &rules)) != ERROR_SUCCESS) { + break; + } + + std::vector scanResults; + const char *nativeString = env->GetStringUTFChars(filePath, 0); + + result = yr_rules_scan_file(rules, nativeString, 0, callback, &scanResults, timeoutSec); + + if (result != ERROR_SUCCESS) { + if (result == ERROR_SCAN_TIMEOUT) { + sprintf_s(errorMessage, "Yara file scan timed out on file %s", nativeString); + } + else { + sprintf_s(errorMessage, "Yara file scan failed (%d)\n", result); + } + throwException(env, errorMessage); + break; + } + + resultList = createArrayList(env, scanResults); + break; + } + + if (rules != NULL) { + yr_rules_destroy(rules); + } + + return resultList; } \ No newline at end of file diff --git a/thirdparty/yara/yarabridge/yarabridge/YaraJNIWrapper.h b/thirdparty/yara/yarabridge/yarabridge/YaraJNIWrapper.h index 6c5f5f5d75..d93db2ac7a 100755 --- a/thirdparty/yara/yarabridge/yarabridge/YaraJNIWrapper.h +++ b/thirdparty/yara/yarabridge/yarabridge/YaraJNIWrapper.h @@ -13,7 +13,15 @@ extern "C" { * Signature: (Ljava/lang/String;[B)Ljava/util/List; */ JNIEXPORT jobject JNICALL Java_org_sleuthkit_autopsy_yara_YaraJNIWrapper_findRuleMatch - (JNIEnv *, jclass, jstring, jbyteArray); + (JNIEnv *, jclass, jstring, jbyteArray, jint, jint); + + /* + * Class: org_sleuthkit_autopsy_yara_YaraJNIWrapper + * Method: findRuleMatchFile + * Signature: (Ljava/lang/String;Ljava/lang/String;)Ljava/util/List; + */ + JNIEXPORT jobject JNICALL Java_org_sleuthkit_autopsy_yara_YaraJNIWrapper_findRuleMatchFile + (JNIEnv *, jclass, jstring, jstring, jint); #ifdef __cplusplus } diff --git a/thirdparty/yara/yarabridge/yarabridge/yarabridge.vcxproj b/thirdparty/yara/yarabridge/yarabridge/yarabridge.vcxproj index ce5dd10c80..c049e81dc9 100755 --- a/thirdparty/yara/yarabridge/yarabridge/yarabridge.vcxproj +++ b/thirdparty/yara/yarabridge/yarabridge/yarabridge.vcxproj @@ -113,7 +113,7 @@ ws2_32.lib;crypt32.lib;libyara64.lib;%(AdditionalDependencies) - copy "$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName).dll" "$(SolutionDir)..\bin\$(ProjectName).dll" + copy "$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName).dll" "$(SolutionDir)..\YaraJNIWrapper\src\org\sleuthkit\autopsy\yara\$(ProjectName).dll" @@ -153,7 +153,7 @@ ws2_32.lib;crypt32.lib;libyara64.lib;%(AdditionalDependencies) - copy "$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName).dll" "$(SolutionDir)..\bin\$(ProjectName).dll" + copy "$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName).dll" "$(SolutionDir)..\YaraJNIWrapper\src\org\sleuthkit\autopsy\yara\$(ProjectName).dll" diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java index d662c5f5e6..886a3bc41f 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java @@ -53,6 +53,7 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.AccountFileInstance; import org.sleuthkit.datamodel.Blackboard; +import org.sleuthkit.datamodel.Blackboard.BlackboardException; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Content; @@ -421,14 +422,16 @@ final class VcardParser { if (attributeType == null) { try{ // Add this attribute type to the case database. - attributeType = tskCase.addArtifactAttributeType(attributeTypeName, + attributeType = tskCase.getBlackboard().getOrAddAttributeType(attributeTypeName, BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, String.format("Phone Number (%s)", StringUtils.capitalize(splitType.toLowerCase()))); - }catch (TskDataException ex) { - attributeType = tskCase.getAttributeType(attributeTypeName); + + ThunderbirdMboxFileIngestModule.addArtifactAttribute(telephoneText, attributeType, attributes); + }catch (BlackboardException ex) { + logger.log(Level.WARNING, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); } } - ThunderbirdMboxFileIngestModule.addArtifactAttribute(telephoneText, attributeType, attributes); + } catch (TskCoreException ex) { logger.log(Level.WARNING, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); } @@ -474,14 +477,14 @@ final class VcardParser { BlackboardAttribute.Type attributeType = tskCase.getAttributeType(attributeTypeName); if (attributeType == null) { // Add this attribute type to the case database. - attributeType = tskCase.addArtifactAttributeType(attributeTypeName, + attributeType = tskCase.getBlackboard().getOrAddAttributeType(attributeTypeName, BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, String.format("Email (%s)", StringUtils.capitalize(splitType.toLowerCase()))); } ThunderbirdMboxFileIngestModule.addArtifactAttribute(email.getValue(), attributeType, attributes); } catch (TskCoreException ex) { logger.log(Level.SEVERE, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); - } catch (TskDataException ex) { + } catch (BlackboardException ex) { logger.log(Level.SEVERE, String.format("Unable to add custom attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); } }