diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/README-POSTGRES-TESTING.md b/Core/src/org/sleuthkit/autopsy/centralrepository/README-POSTGRES-TESTING.md deleted file mode 100755 index ed202b3fe5..0000000000 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/README-POSTGRES-TESTING.md +++ /dev/null @@ -1,77 +0,0 @@ -# Setting up the PostgreSQL DB for use in Enterprise Artifact Manager - -## Using Command Line (cmd.exe) - -The easiest way to do this is with the scripts that come with the PostgreSQL server. -Add the PostgreSQL Server bin directory to your path in your user's environment -variables. - - PATH=$PATH;c:\Program Files\PostgreSQL\9.6\bin - -Note: I've had issues getting these Windows PostgreSQL binaries to run in -a cygwin terminal. But they work perfectly in cmd.exe. - -### Create Role - -The role we use has user name "testuser" and password "testpass". - - $ createuser -U postgres -P testuser - -When prompted for a password enter "testpass". - -### Create Database - -The database we use is named "enterpriseartifactmanagerdb". - - $ createdb -T template0 -U postgres -O testuser enterpriseartifactmanagerdb - -### Drop Database - -If we want to reset the database, it's easiest to just drop it. - - $ dropdb -U postgres enterpriseartifactmanagerdb - -### Load the database content from a .sql file - -Before loading a schema.sql file, you must have an empty database named -enterpriseartifactmanagerdb and an existing user named testuser that is the database owner. -Use the schema.sql files to create the tables, indices, and other required settings. - - $ psql -U postgres enterpriseartifactmanagerdb < c:\path\to\schema.sql - -## Using pgAdmin tool - -### Create Role - -Use the right-click menus to create a role named "testuser" with a password of -"testpass". - -### Create Database - -Use the right-click menus to create a database named "enterpriseartifactmanagerdb" -and set testuser as the owner. - -### Load the database content from a .sql file - -Right-click on the enterpriseartifactmanagerdb database and select "CREATE script". -In the new window, delete all of the content and paste in the content of the -schema.sql file. -Click on the lightning icon to execute the contents you just pasted. - -## Notes - -- The schema.sql file does not contain commands to check for the existence of -existing table objects, nor does it drop them before trying to add new ones. -So, it is best to drop and create the database freshly before loading the schema. -- pg_restore cannot load a .sql file. -- The schema.sql file cannot have commands to drop/create the database. - -### Purpose of each schema file - -schema1 - 2 non-normalized tables, no FKs, no enforced uniqueness - -schema2 - 2 non-normalized tables, no FKs, enforced uniqueness on artifacts -- requires PostgreSQL Server ver 9.2+ -- postgresql automatically creates a unique index when a unique constraint is defined, -so there is no need to manually create a unique index for the same column(s). -Doing so would duplicate the automatically-created index. \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/README_MONGODB_TESTING.md b/Core/src/org/sleuthkit/autopsy/centralrepository/README_MONGODB_TESTING.md deleted file mode 100755 index 73c1a33ce1..0000000000 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/README_MONGODB_TESTING.md +++ /dev/null @@ -1,168 +0,0 @@ -# Mongo Database for testing - -These instructions assume that you already have MongoDB Server 3.4 installed and -that the server binary is at: - - C:\Program Files\MongoDB\Server\3.4\bin\mongod.exe - -## Using multiple configurations - -It is possible to use a configuration file and even to run MongoDB as a service, -but for our testing, we want one instance that is the default non-secure instance. -And we want a second instance that has auth enabled and has a defined role for -our test user. - -The the easiest way to have multiple instances of MongoDB that have -unique configurations is to have each of them use distinct directories. - -The default directories are: - - C:\data\db - C:\data\log - -So, we need to create a second set of directories at: - - C:\dataauth\db - C:\dataauth\log - -### Create config files, so it is easier to start MongoDB - -The config file is YAML formatted, so do not use TABs. And every -sub-level should be indented 2 spaces from the previous level. - -We will not define the host/port values in the config files, because we assume -that you are using the default port and host values AND -that you will only run ONE of these two instances at a time. -If you want them to run at the same time, include the net.port and net.bindIp -parameters in the config file and make sure they are not both using the same -port/IP pair. - -The first config file is for the default instance, create a new file: - - C:\data\mongod.cfg - -Enter the following content into that file: - - systemLog: - destination: file - path: c:\data\log\mongod.log - storage: - dbPath: c:\data\db - -The second config file is for the auth instance, create a new file: - - C:\dataauth\mongod.cfg - -Enter the following content into that file: - - systemLog: - destination: file - path: c:\dataauth\log\mongod.log - storage: - dbPath: c:\dataauth\db - security: - authorization: enabled - -Note: Ensure Windows did not name your file ending with cfg.txt. You'll -have to go to Folder Options -> View and uncheck the option to hide file extensions -for common files. - -Also, if you will be running mongod using a windows terminal (cmd.exe or powershell), -make sure to use correct Windows path separators (i.e. c:\data\db). -If you are instead using cygwin, make sure to use valid cygwin/unix path -separators (i.e. c:/data/db). - -### To start MongoDB using a config file - -for Windows cmd or ps: - - C:\path\to\bin\mongod.exe --config C:\data\mongod.cfg - -or - - C:\path\to\bin\mongod.exe --config C:\dataauth\mongod.cfg - -for cygwin/unix: - - cd /cygdrive/c/path/to/bin/ - ./mongod.exe --config C:/data/mongod.cfg - -or - - ./mongod.exe --config C:/dataauth/mongod.cfg - - -If it starts correctly, you'll see nothing in the terminal and all logs will -go to the specified log file. -If there is a problem it will display an error in the terminal and fail to start. - -### Setting up the auth'd instance - -The first time you start this mongod instance, you MUST start with auth disabled, -so it will let you create the admin user. Do this in the config file: - - security: - authorization: disabled - -Now start the auth'd mongod in one terminal. - -#### Create admin user - -In a second terminal, connect to that instance with mongo client. - - C:\path\to\mongo.exe - -Enter the following in the mongo client: - - use admin - db.createUser( - { - user: "adminuser", - pwd: "adminpass", - roles: [ { role: "userAdminAnyDatabase", db: "admin" } ] - } - ) - -Logout of the mongo client. -Stop mongod. -Set security.authorization to enabled in the auth'd mongod.cfg. -Start the auth'd mongod. -From now on, you can always start this instance of mongod with authorization -enabled. - -#### Create test user - -In the second terminal, connect to the auth'd instance with mongo client. - - C:\path\to\mongo.exe - -Authenticate as the admin user: - - use admin - db.auth("adminuser", "adminpass") - -Create the test user in the enterpriseartifactmanagerdb database with the readWrite role: - - use enterpriseartifactmanagerdb - db.createUser( - { - user: "testuser", - pwd: "testpass", - roles: [ { role: "readWrite", db: "enterpriseartifactmanagerdb" } ] - } - ) - -Now the EnterpriseArtifactManager code can use the user "testuser" to use the -enterpriseartifactmanagerdb database. This includes creating/dropping indices/collections -along with the usual insert/update/delete commands. - -NOTE: The database where a user is created is that user's "authentication database". -So, when that user needs to authenticate, they need to provide their username, -password, and authentication database. - -### References - -Installation and setup: https://docs.mongodb.com/manual/tutorial/install-mongodb-on-windows/ -Config file: https://docs.mongodb.com/manual/reference/configuration-options/ -Enabling Auth: https://docs.mongodb.com/manual/tutorial/enable-authentication/ -readWrite Role: https://docs.mongodb.com/manual/reference/built-in-roles/#readWrite \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/Bundle.properties b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/Bundle.properties index f1ab298155..8b18108d52 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/Bundle.properties @@ -2,4 +2,4 @@ DataContentViewerOtherCases.selectAllMenuItem.text=Select All DataContentViewerOtherCases.showCaseDetailsMenuItem.text=Show Case Details DataContentViewerOtherCases.table.toolTip.text=Click column name to sort. Right-click on the table for more options. DataContentViewerOtherCases.exportToCSVMenuItem.text=Export Selected Rows to CSV -DataContentViewerOtherCases.showCommonalityMenuItem.text=Show Commonality +DataContentViewerOtherCases.showCommonalityMenuItem.text=Show Frequency diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java index 8ab241fa3b..ac71624da7 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java @@ -47,8 +47,8 @@ import org.openide.util.NbBundle.Messages; import org.openide.util.lookup.ServiceProvider; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.corecomponentinterfaces.DataContentViewer; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifact; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactInstance; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttribute; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil; import org.sleuthkit.autopsy.centralrepository.datamodel.EamCase; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; @@ -73,18 +73,18 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D private final static Logger LOGGER = Logger.getLogger(DataContentViewerOtherCases.class.getName()); private final DataContentViewerOtherCasesTableModel tableModel; - private final Collection correlatedArtifacts; + private final Collection correlationAttributes; /** * Creates new form DataContentViewerOtherCases */ public DataContentViewerOtherCases() { this.tableModel = new DataContentViewerOtherCasesTableModel(); - this.correlatedArtifacts = new ArrayList<>(); + this.correlationAttributes = new ArrayList<>(); initComponents(); customizeComponents(); - readSettings(); + reset(); } private void customizeComponents() { @@ -119,11 +119,14 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D "# {0} - commonality percentage", "# {1} - correlation type", "# {2} - correlation value", - "DataContentViewerOtherCases.correlatedArtifacts.byType={0}% for Correlation Type: {1} and Correlation Value: {2}.\n", - "DataContentViewerOtherCases.correlatedArtifacts.title=Commonality Percentages", - "DataContentViewerOtherCases.correlatedArtifacts.failed=Failed to get commonality details."}) + "DataContentViewerOtherCases.correlatedArtifacts.byType={0}% of data sources have {2} (type: {1})\n", + "DataContentViewerOtherCases.correlatedArtifacts.title=Attribute Frequency", + "DataContentViewerOtherCases.correlatedArtifacts.failed=Failed to get frequency details."}) + /** + * Show how common the selected + */ private void showCommonalityDetails() { - if (correlatedArtifacts.isEmpty()) { + if (correlationAttributes.isEmpty()) { JOptionPane.showConfirmDialog(showCommonalityMenuItem, Bundle.DataContentViewerOtherCases_correlatedArtifacts_isEmpty(), Bundle.DataContentViewerOtherCases_correlatedArtifacts_title(), @@ -133,8 +136,8 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D int percentage; try { EamDb dbManager = EamDb.getInstance(); - for (EamArtifact eamArtifact : correlatedArtifacts) { - percentage = dbManager.getCommonalityPercentageForTypeValue(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue()); + for (CorrelationAttribute eamArtifact : correlationAttributes) { + percentage = dbManager.getFrequencyPercentage(eamArtifact); msg.append(Bundle.DataContentViewerOtherCases_correlatedArtifacts_byType(percentage, eamArtifact.getCorrelationType().getDisplayName(), eamArtifact.getCorrelationValue())); @@ -163,7 +166,7 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D if (-1 != selectedRowViewIdx) { EamDb dbManager = EamDb.getInstance(); int selectedRowModelIdx = otherCasesTable.convertRowIndexToModel(selectedRowViewIdx); - EamArtifact eamArtifact = (EamArtifact) tableModel.getRow(selectedRowModelIdx); + CorrelationAttribute eamArtifact = (CorrelationAttribute) tableModel.getRow(selectedRowModelIdx); EamCase eamCasePartial = eamArtifact.getInstances().get(0).getEamCase(); if (eamCasePartial == null) { JOptionPane.showConfirmDialog(showCaseDetailsMenuItem, @@ -262,14 +265,12 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D } /** - * Read the module settings from the config file and reset the table model. + * Reset the UI and clear cached data. */ - private boolean readSettings() { + private void reset() { // start with empty table tableModel.clearTable(); - correlatedArtifacts.clear(); - - return true; + correlationAttributes.clear(); } @Override @@ -294,7 +295,7 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D @Override public void resetComponent() { - readSettings(); + reset(); } @Override @@ -365,102 +366,61 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D } /** - * Scan a Node for blackboard artifacts / content that we can correlate on - * and create the corresponding Central Repository artifacts for display + * Determine what attributes can be used for correlation based on the node. * - * @param node The node to view + * @param node The node to correlate * - * @return A collection of central repository artifacts to display + * @return A list of attributes that can be used for correlation */ - private Collection getArtifactsFromCorrelatableAttributes(Node node) { - Collection ret = new ArrayList<>(); + private Collection getCorrelationAttributesFromNode(Node node) { + Collection ret = new ArrayList<>(); - /* - * If the user selected a blackboard artifact or tag of a BB artifact, - * correlate both the artifact and the associated file. If the user - * selected a file, correlate only the file - */ - BlackboardArtifact bbArtifact = getBlackboardArtifactFromNode(node); - AbstractFile abstractFile = getAbstractFileFromNode(node); - List artifactTypes = null; - try { - EamDb dbManager = EamDb.getInstance(); - artifactTypes = dbManager.getCorrelationTypes(); - if (bbArtifact != null) { - ret.addAll(EamArtifactUtil.fromBlackboardArtifact(bbArtifact, false, artifactTypes, false)); - } - } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error retrieving correlation types", ex); // NON-NLS + // correlate on blackboard artifact attributes if they exist and supported + BlackboardArtifact bbArtifact = getBlackboardArtifactFromNode(node); + if (bbArtifact != null) { + ret.addAll(EamArtifactUtil.getCorrelationAttributeFromBlackboardArtifact(bbArtifact, false, false)); } - + + // we can correlate based on the MD5 if it is enabled + AbstractFile abstractFile = getAbstractFileFromNode(node); if (abstractFile != null) { - String md5 = abstractFile.getMd5Hash(); - if (md5 != null && !md5.isEmpty() && null != artifactTypes && !artifactTypes.isEmpty()) { - for (EamArtifact.Type aType : artifactTypes) { - if (aType.getId() == EamArtifact.FILES_TYPE_ID) { - ret.add(new EamArtifact(aType, md5)); - break; + try { + List artifactTypes = EamDb.getInstance().getDefinedCorrelationTypes(); + String md5 = abstractFile.getMd5Hash(); + if (md5 != null && !md5.isEmpty() && null != artifactTypes && !artifactTypes.isEmpty()) { + for (CorrelationAttribute.Type aType : artifactTypes) { + if (aType.getId() == CorrelationAttribute.FILES_TYPE_ID) { + ret.add(new CorrelationAttribute(aType, md5)); + break; + } } } + } catch (EamDbException ex) { + LOGGER.log(Level.SEVERE, "Error connecting to DB", ex); // NON-NLS } } return ret; } - /** - * Given a node, return the associated data source - * - * @param node The node - * - * @return The name of the data source - */ - private String getDataSourceNameFromNode(Node node) { - AbstractFile af = getAbstractFileFromNode(node); - try { - if (af != null) { - return af.getDataSource().getName(); - } - } catch (TskException ex) { - return ""; - } - return ""; - } - - /** - * Given a node, return the associated data source's device ID - * - * @param node The node - * - * @return The ID of the data source's device - */ - private String getDeviceIdFromNode(Node node) { - AbstractFile af = getAbstractFileFromNode(node); - try { - if (af != null) { - return Case.getCurrentCase().getSleuthkitCase().getDataSource(af.getDataSource().getId()).getDeviceId(); - } - } catch (TskException ex) { - return ""; - } - - return ""; - } /** * Query the db for artifact instances from other cases correlated to the - * given central repository artifact. + * given central repository artifact. Will not show instances from the same datasource / device * - * @param eamArtifact The artifact to correlate against + * @param corAttr CorrelationAttribute to query for + * @param dataSourceName Data source to filter results + * @param deviceId Device Id to filter results * * @return A collection of correlated artifact instances from other cases */ - private Collection getCorrelatedInstances(EamArtifact.Type aType, String value, String dataSourceName, String deviceId) { + private Collection getCorrelatedInstances(CorrelationAttribute corAttr, String dataSourceName, String deviceId) { + // @@@ Check exception String caseUUID = Case.getCurrentCase().getName(); try { EamDb dbManager = EamDb.getInstance(); - Collection artifactInstances = dbManager.getArtifactInstancesByTypeValue(aType, value).stream() + Collection artifactInstances = dbManager.getArtifactInstancesByTypeValue(corAttr.getCorrelationType(), corAttr.getCorrelationValue()).stream() .filter(artifactInstance -> !artifactInstance.getEamCase().getCaseUUID().equals(caseUUID) || !artifactInstance.getEamDataSource().getName().equals(dataSourceName) || !artifactInstance.getEamDataSource().getDeviceID().equals(deviceId)) @@ -475,25 +435,25 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D /** * Get the global file instances matching the given eamArtifact and convert - * them to central repository artifact instancess. + * them to central repository artifact instances. * * @param eamArtifact Artifact to use for ArtifactTypeEnum matching * * @return List of central repository artifact instances, empty list if none * found */ - public Collection getReferenceInstancesAsArtifactInstances(EamArtifact eamArtifact) { - Collection eamArtifactInstances = new ArrayList<>(); + public Collection getReferenceInstancesAsArtifactInstances(CorrelationAttribute eamArtifact) { + Collection eamArtifactInstances = new ArrayList<>(); // FUTURE: support other reference types - if (eamArtifact.getCorrelationType().getId() != EamArtifact.FILES_TYPE_ID) { + if (eamArtifact.getCorrelationType().getId() != CorrelationAttribute.FILES_TYPE_ID) { return Collections.emptyList(); } try { EamDb dbManager = EamDb.getInstance(); Collection eamGlobalFileInstances = dbManager.getReferenceInstancesByTypeValue(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue()); eamGlobalFileInstances.forEach((eamGlobalFileInstance) -> { - eamArtifactInstances.add(new EamArtifactInstance( - null, null, "", eamGlobalFileInstance.getComment(), eamGlobalFileInstance.getKnownStatus(), EamArtifactInstance.GlobalStatus.GLOBAL + eamArtifactInstances.add(new CorrelationAttributeInstance( + null, null, "", eamGlobalFileInstance.getComment(), eamGlobalFileInstance.getKnownStatus(), CorrelationAttributeInstance.GlobalStatus.GLOBAL )); }); return eamArtifactInstances; @@ -510,7 +470,7 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D } // Is supported if this node has correlatable content (File, BlackboardArtifact) - return !getArtifactsFromCorrelatableAttributes(node).isEmpty(); + return !getCorrelationAttributesFromNode(node).isEmpty(); } @Override @@ -520,7 +480,10 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D return; } - readSettings(); // reset the table to empty. + reset(); // reset the table to empty. + if (node == null) { + return; + } populateTable(node); } @@ -533,26 +496,40 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D @Messages({"DataContentViewerOtherCases.table.isempty=There are no associated artifacts or files from other occurrences to display.", "DataContentViewerOtherCases.table.noArtifacts=Correlation cannot be performed on the selected file."}) private void populateTable(Node node) { - String dataSourceName = getDataSourceNameFromNode(node); - String deviceId = getDeviceIdFromNode(node); - correlatedArtifacts.addAll(getArtifactsFromCorrelatableAttributes(node)); - correlatedArtifacts.forEach((eamArtifact) -> { - // get local instances - Collection eamArtifactInstances = getCorrelatedInstances(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue(), dataSourceName, deviceId); - // get global instances - eamArtifactInstances.addAll(getReferenceInstancesAsArtifactInstances(eamArtifact)); + AbstractFile af = getAbstractFileFromNode(node); + String dataSourceName = ""; + String deviceId = ""; + try { + if (af != null) { + Content dataSource = af.getDataSource(); + dataSourceName = dataSource.getName(); + deviceId = Case.getCurrentCase().getSleuthkitCase().getDataSource(dataSource.getId()).getDeviceId(); + } + } catch (TskException ex) { + // do nothing. + // @@@ Review this behavior + } + + // get the attributes we can correlate on + correlationAttributes.addAll(getCorrelationAttributesFromNode(node)); + for (CorrelationAttribute corAttr : correlationAttributes) { + Collection corAttrInstances = new ArrayList<>(); + + // get correlation and reference set instances from DB + corAttrInstances.addAll(getCorrelatedInstances(corAttr, dataSourceName, deviceId)); + corAttrInstances.addAll(getReferenceInstancesAsArtifactInstances(corAttr)); - eamArtifactInstances.forEach((eamArtifactInstance) -> { - EamArtifact newCeArtifact = new EamArtifact( - eamArtifact.getCorrelationType(), - eamArtifact.getCorrelationValue() + corAttrInstances.forEach((corAttrInstance) -> { + CorrelationAttribute newCeArtifact = new CorrelationAttribute( + corAttr.getCorrelationType(), + corAttr.getCorrelationValue() ); - newCeArtifact.addInstance(eamArtifactInstance); + newCeArtifact.addInstance(corAttrInstance); tableModel.addEamArtifact(newCeArtifact); }); - }); + } - if (correlatedArtifacts.isEmpty()) { + if (correlationAttributes.isEmpty()) { displayMessageOnTableStatusPanel(Bundle.DataContentViewerOtherCases_table_noArtifacts()); } else if (0 == tableModel.getRowCount()) { displayMessageOnTableStatusPanel(Bundle.DataContentViewerOtherCases_table_isempty()); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCasesTableCellRenderer.java b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCasesTableCellRenderer.java index 3a657e3d02..0c2968fe6b 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCasesTableCellRenderer.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCasesTableCellRenderer.java @@ -55,7 +55,8 @@ public class DataContentViewerOtherCasesTableCellRenderer implements TableCellRe background = Color.RED; } else if (known_status.equals(TskData.FileKnown.UNKNOWN.getName())) { foreground = Color.BLACK; - background = Color.YELLOW; + //background = Color.YELLOW; + background = Color.WHITE; } else { foreground = Color.BLACK; background = Color.WHITE; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCasesTableModel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCasesTableModel.java index ba2358d75c..b27b6880d6 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCasesTableModel.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCasesTableModel.java @@ -22,8 +22,8 @@ import java.util.ArrayList; import java.util.List; import javax.swing.table.AbstractTableModel; import org.openide.util.NbBundle.Messages; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifact; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactInstance; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttribute; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; /** * Model for cells in data content viewer table @@ -45,13 +45,13 @@ public class DataContentViewerOtherCasesTableModel extends AbstractTableModel { // If order is changed, update the CellRenderer to ensure correct row coloring. CASE_NAME(Bundle.DataContentViewerOtherCasesTableModel_case(), 75), DATA_SOURCE(Bundle.DataContentViewerOtherCasesTableModel_dataSource(), 75), - DEVICE(Bundle.DataContentViewerOtherCasesTableModel_device(), 145), TYPE(Bundle.DataContentViewerOtherCasesTableModel_type(), 40), VALUE(Bundle.DataContentViewerOtherCasesTableModel_value(), 145), KNOWN(Bundle.DataContentViewerOtherCasesTableModel_known(), 45), SCOPE(Bundle.DataContentViewerOtherCasesTableModel_scope(), 20), COMMENT(Bundle.DataContentViewerOtherCasesTableModel_comment(), 200), - FILE_PATH(Bundle.DataContentViewerOtherCasesTableModel_path(), 250); + FILE_PATH(Bundle.DataContentViewerOtherCasesTableModel_path(), 250), + DEVICE(Bundle.DataContentViewerOtherCasesTableModel_device(), 145); private final String columnName; private final int columnWidth; @@ -70,7 +70,7 @@ public class DataContentViewerOtherCasesTableModel extends AbstractTableModel { } }; - List eamArtifacts; + List eamArtifacts; DataContentViewerOtherCasesTableModel() { eamArtifacts = new ArrayList<>(); @@ -127,8 +127,8 @@ public class DataContentViewerOtherCasesTableModel extends AbstractTableModel { * @return value in the cell */ private Object mapValueById(int rowIdx, TableColumns colId) { - EamArtifact eamArtifact = eamArtifacts.get(rowIdx); - EamArtifactInstance eamArtifactInstance = eamArtifact.getInstances().get(0); + CorrelationAttribute eamArtifact = eamArtifacts.get(rowIdx); + CorrelationAttributeInstance eamArtifactInstance = eamArtifact.getInstances().get(0); String value = Bundle.DataContentViewerOtherCasesTableModel_noData(); switch (colId) { @@ -180,13 +180,14 @@ public class DataContentViewerOtherCasesTableModel extends AbstractTableModel { * @param eamArtifact central repository artifact to add to the * table */ - public void addEamArtifact(EamArtifact eamArtifact) { + public void addEamArtifact(CorrelationAttribute eamArtifact) { eamArtifacts.add(eamArtifact); fireTableDataChanged(); } public void clearTable() { eamArtifacts.clear(); + fireTableDataChanged(); } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index cae074db95..accd263d57 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -47,11 +47,11 @@ public abstract class AbstractSqlEamDb implements EamDb { private final static Logger LOGGER = Logger.getLogger(AbstractSqlEamDb.class.getName()); - protected final List DEFAULT_CORRELATION_TYPES; + protected final List DEFAULT_CORRELATION_TYPES; private int bulkArtifactsCount; protected int bulkArtifactsThreshold; - private final Map> bulkArtifacts; + private final Map> bulkArtifacts; private final List badTags; /** @@ -64,7 +64,7 @@ public abstract class AbstractSqlEamDb implements EamDb { bulkArtifactsCount = 0; bulkArtifacts = new HashMap<>(); - DEFAULT_CORRELATION_TYPES = EamArtifact.getDefaultCorrelationTypes(); + DEFAULT_CORRELATION_TYPES = CorrelationAttribute.getDefaultCorrelationTypes(); DEFAULT_CORRELATION_TYPES.forEach((type) -> { bulkArtifacts.put(type.getDbTableName(), new ArrayList<>()); }); @@ -423,7 +423,7 @@ public abstract class AbstractSqlEamDb implements EamDb { * @param eamDataSource the data source to add */ @Override - public void newDataSource(EamDataSource eamDataSource) throws EamDbException { + public void newDataSource(CorrelationDataSource eamDataSource) throws EamDbException { Connection conn = connect(); PreparedStatement preparedStatement = null; @@ -451,7 +451,7 @@ public abstract class AbstractSqlEamDb implements EamDb { * @param eamDataSource the data source to update */ @Override - public void updateDataSource(EamDataSource eamDataSource) throws EamDbException { + public void updateDataSource(CorrelationDataSource eamDataSource) throws EamDbException { Connection conn = connect(); PreparedStatement preparedStatement = null; @@ -480,10 +480,10 @@ public abstract class AbstractSqlEamDb implements EamDb { * @return The data source */ @Override - public EamDataSource getDataSourceDetails(String dataSourceDeviceId) throws EamDbException { + public CorrelationDataSource getDataSourceDetails(String dataSourceDeviceId) throws EamDbException { Connection conn = connect(); - EamDataSource eamDataSourceResult = null; + CorrelationDataSource eamDataSourceResult = null; PreparedStatement preparedStatement = null; ResultSet resultSet = null; @@ -513,11 +513,11 @@ public abstract class AbstractSqlEamDb implements EamDb { * @return list of data sources in the DB */ @Override - public List getDataSources() throws EamDbException { + public List getDataSources() throws EamDbException { Connection conn = connect(); - List dataSources = new ArrayList<>(); - EamDataSource eamDataSourceResult; + List dataSources = new ArrayList<>(); + CorrelationDataSource eamDataSourceResult; PreparedStatement preparedStatement = null; ResultSet resultSet = null; @@ -548,12 +548,14 @@ public abstract class AbstractSqlEamDb implements EamDb { * @param eamArtifact The artifact to add */ @Override - public void addArtifact(EamArtifact eamArtifact) throws EamDbException { + public void addArtifact(CorrelationAttribute eamArtifact) throws EamDbException { Connection conn = connect(); - List eamInstances = eamArtifact.getInstances(); + List eamInstances = eamArtifact.getInstances(); PreparedStatement preparedStatement = null; + + // @@@ We should cache the case and data source IDs in memory String tableName = EamDbUtil.correlationTypeToInstanceTableName(eamArtifact.getCorrelationType()); StringBuilder sql = new StringBuilder(); sql.append("INSERT INTO "); @@ -564,7 +566,7 @@ public abstract class AbstractSqlEamDb implements EamDb { try { preparedStatement = conn.prepareStatement(sql.toString()); - for (EamArtifactInstance eamInstance : eamInstances) { + for (CorrelationAttributeInstance eamInstance : eamInstances) { if(! eamArtifact.getCorrelationValue().isEmpty()){ preparedStatement.setString(1, eamInstance.getEamCase().getCaseUUID()); preparedStatement.setString(2, eamInstance.getEamDataSource().getDeviceID()); @@ -597,12 +599,12 @@ public abstract class AbstractSqlEamDb implements EamDb { * @return List of artifact instances for a given type/value */ @Override - public List getArtifactInstancesByTypeValue(EamArtifact.Type aType, String value) throws EamDbException { + public List getArtifactInstancesByTypeValue(CorrelationAttribute.Type aType, String value) throws EamDbException { Connection conn = connect(); - List artifactInstances = new ArrayList<>(); + List artifactInstances = new ArrayList<>(); - EamArtifactInstance artifactInstance; + CorrelationAttributeInstance artifactInstance; PreparedStatement preparedStatement = null; ResultSet resultSet = null; @@ -649,12 +651,12 @@ public abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ @Override - public List getArtifactInstancesByPath(EamArtifact.Type aType, String filePath) throws EamDbException { + public List getArtifactInstancesByPath(CorrelationAttribute.Type aType, String filePath) throws EamDbException { Connection conn = connect(); - List artifactInstances = new ArrayList<>(); + List artifactInstances = new ArrayList<>(); - EamArtifactInstance artifactInstance; + CorrelationAttributeInstance artifactInstance; PreparedStatement preparedStatement = null; ResultSet resultSet = null; @@ -700,7 +702,7 @@ public abstract class AbstractSqlEamDb implements EamDb { * ArtifactValue. */ @Override - public Long getCountArtifactInstancesByTypeValue(EamArtifact.Type aType, String value) throws EamDbException { + public Long getCountArtifactInstancesByTypeValue(CorrelationAttribute.Type aType, String value) throws EamDbException { Connection conn = connect(); Long instanceCount = 0L; @@ -730,21 +732,11 @@ public abstract class AbstractSqlEamDb implements EamDb { return instanceCount; } - /** - * Using the ArtifactType and ArtifactValue from the given eamArtfact, - * compute the ratio of: (The number of unique case_id/datasource_id tuples - * where Type/Value is found) divided by (The total number of unique - * case_id/datasource_id tuples in the database) expressed as a percentage. - * - * @param eamArtifact Artifact with artifactType and artifactValue to search - * for - * - * @return Int between 0 and 100 - */ + @Override - public int getCommonalityPercentageForTypeValue(EamArtifact.Type aType, String value) throws EamDbException { - Double uniqueTypeValueTuples = getCountUniqueCaseDataSourceTuplesHavingTypeValue(aType, value).doubleValue(); - Double uniqueCaseDataSourceTuples = getCountUniqueCaseDataSourceTuples().doubleValue(); + public int getFrequencyPercentage(CorrelationAttribute corAttr) throws EamDbException { + Double uniqueTypeValueTuples = getCountUniqueCaseDataSourceTuplesHavingTypeValue(corAttr.getCorrelationType(), corAttr.getCorrelationValue()).doubleValue(); + Double uniqueCaseDataSourceTuples = getCountUniqueDataSources().doubleValue(); Double commonalityPercentage = uniqueTypeValueTuples / uniqueCaseDataSourceTuples * 100; return commonalityPercentage.intValue(); } @@ -760,7 +752,7 @@ public abstract class AbstractSqlEamDb implements EamDb { * @return Number of unique tuples */ @Override - public Long getCountUniqueCaseDataSourceTuplesHavingTypeValue(EamArtifact.Type aType, String value) throws EamDbException { + public Long getCountUniqueCaseDataSourceTuplesHavingTypeValue(CorrelationAttribute.Type aType, String value) throws EamDbException { Connection conn = connect(); Long instanceCount = 0L; @@ -792,41 +784,24 @@ public abstract class AbstractSqlEamDb implements EamDb { return instanceCount; } - /** - * Retrieves number of unique caseDisplayName/dataSource tuples in the - * database. - * - * @return Number of unique tuples - */ + @Override - public Long getCountUniqueCaseDataSourceTuples() throws EamDbException { + public Long getCountUniqueDataSources() throws EamDbException { Connection conn = connect(); Long instanceCount = 0L; - List artifactTypes = getCorrelationTypes(); PreparedStatement preparedStatement = null; ResultSet resultSet = null; - StringBuilder sql = new StringBuilder(); - sql.append("SELECT 0 "); - - for (EamArtifact.Type type : artifactTypes) { - String table_name = EamDbUtil.correlationTypeToInstanceTableName(type); - - sql.append("+ (SELECT count(*) FROM (SELECT DISTINCT case_id, data_source_id FROM "); - sql.append(table_name); - sql.append(") AS "); - sql.append(table_name); - sql.append("_distinct_case_data_source_tuple) "); - } + String stmt = "SELECT count(*) FROM data_sources"; try { - preparedStatement = conn.prepareStatement(sql.toString()); + preparedStatement = conn.prepareStatement(stmt); resultSet = preparedStatement.executeQuery(); resultSet.next(); instanceCount = resultSet.getLong(1); } catch (SQLException ex) { - throw new EamDbException("Error counting unique caseDisplayName/dataSource tuples.", ex); // NON-NLS + throw new EamDbException("Error counting data sources.", ex); // NON-NLS } finally { EamDbUtil.closePreparedStatement(preparedStatement); EamDbUtil.closeResultSet(resultSet); @@ -852,7 +827,7 @@ public abstract class AbstractSqlEamDb implements EamDb { Connection conn = connect(); Long instanceCount = 0L; - List artifactTypes = getCorrelationTypes(); + List artifactTypes = getDefinedCorrelationTypes(); PreparedStatement preparedStatement = null; ResultSet resultSet = null; @@ -860,7 +835,7 @@ public abstract class AbstractSqlEamDb implements EamDb { StringBuilder sql = new StringBuilder(); sql.append("SELECT 0 "); - for (EamArtifact.Type type : artifactTypes) { + for (CorrelationAttribute.Type type : artifactTypes) { String table_name = EamDbUtil.correlationTypeToInstanceTableName(type); sql.append("+ (SELECT count(*) FROM "); @@ -898,7 +873,7 @@ public abstract class AbstractSqlEamDb implements EamDb { * @param eamArtifact The artifact to add */ @Override - public void prepareBulkArtifact(EamArtifact eamArtifact) throws EamDbException { + public void prepareBulkArtifact(CorrelationAttribute eamArtifact) throws EamDbException { synchronized (bulkArtifacts) { bulkArtifacts.get(eamArtifact.getCorrelationType().getDbTableName()).add(eamArtifact); @@ -923,7 +898,7 @@ public abstract class AbstractSqlEamDb implements EamDb { */ @Override public void bulkInsertArtifacts() throws EamDbException { - List artifactTypes = getCorrelationTypes(); + List artifactTypes = getDefinedCorrelationTypes(); Connection conn = connect(); PreparedStatement bulkPs = null; @@ -934,7 +909,7 @@ public abstract class AbstractSqlEamDb implements EamDb { return; } - for (EamArtifact.Type type : artifactTypes) { + for (CorrelationAttribute.Type type : artifactTypes) { String tableName = EamDbUtil.correlationTypeToInstanceTableName(type); StringBuilder sql = new StringBuilder(); @@ -947,11 +922,11 @@ public abstract class AbstractSqlEamDb implements EamDb { bulkPs = conn.prepareStatement(sql.toString()); - Collection eamArtifacts = bulkArtifacts.get(type.getDbTableName()); - for (EamArtifact eamArtifact : eamArtifacts) { - List eamInstances = eamArtifact.getInstances(); + Collection eamArtifacts = bulkArtifacts.get(type.getDbTableName()); + for (CorrelationAttribute eamArtifact : eamArtifacts) { + List eamInstances = eamArtifact.getInstances(); - for (EamArtifactInstance eamInstance : eamInstances) { + for (CorrelationAttributeInstance eamInstance : eamInstances) { if(! eamArtifact.getCorrelationValue().isEmpty()){ bulkPs.setString(1, eamInstance.getEamCase().getCaseUUID()); bulkPs.setString(2, eamInstance.getEamDataSource().getDeviceID()); @@ -1068,15 +1043,15 @@ public abstract class AbstractSqlEamDb implements EamDb { * @param FileKnown The status to change the artifact to */ @Override - public void setArtifactInstanceKnownStatus(EamArtifact eamArtifact, TskData.FileKnown knownStatus) throws EamDbException { + public void setArtifactInstanceKnownStatus(CorrelationAttribute eamArtifact, TskData.FileKnown knownStatus) throws EamDbException { Connection conn = connect(); if (1 != eamArtifact.getInstances().size()) { throw new EamDbException("Error: Artifact must have exactly one (1) Artifact Instance to set as notable."); // NON-NLS } - List eamInstances = eamArtifact.getInstances(); - EamArtifactInstance eamInstance = eamInstances.get(0); + List eamInstances = eamArtifact.getInstances(); + CorrelationAttributeInstance eamInstance = eamInstances.get(0); PreparedStatement preparedUpdate = null; PreparedStatement preparedQuery = null; @@ -1160,12 +1135,12 @@ public abstract class AbstractSqlEamDb implements EamDb { * @return List with 0 or more matching eamArtifact instances. */ @Override - public List getArtifactInstancesKnownBad(EamArtifact.Type aType, String value) throws EamDbException { + public List getArtifactInstancesKnownBad(CorrelationAttribute.Type aType, String value) throws EamDbException { Connection conn = connect(); - List artifactInstances = new ArrayList<>(); + List artifactInstances = new ArrayList<>(); - EamArtifactInstance artifactInstance; + CorrelationAttributeInstance artifactInstance; PreparedStatement preparedStatement = null; ResultSet resultSet = null; @@ -1210,7 +1185,7 @@ public abstract class AbstractSqlEamDb implements EamDb { * @return Number of matching eamArtifacts */ @Override - public Long getCountArtifactInstancesKnownBad(EamArtifact.Type aType, String value) throws EamDbException { + public Long getCountArtifactInstancesKnownBad(CorrelationAttribute.Type aType, String value) throws EamDbException { Connection conn = connect(); Long badInstances = 0L; @@ -1254,7 +1229,7 @@ public abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ @Override - public List getListCasesHavingArtifactInstancesKnownBad(EamArtifact.Type aType, String value) throws EamDbException { + public List getListCasesHavingArtifactInstancesKnownBad(CorrelationAttribute.Type aType, String value) throws EamDbException { Connection conn = connect(); Collection caseNames = new LinkedHashSet<>(); @@ -1302,10 +1277,10 @@ public abstract class AbstractSqlEamDb implements EamDb { * @return Global known status of the artifact */ @Override - public boolean isArtifactlKnownBadByReference(EamArtifact.Type aType, String value) throws EamDbException { + public boolean isArtifactlKnownBadByReference(CorrelationAttribute.Type aType, String value) throws EamDbException { // TEMP: Only support file correlation type - if (aType.getId() != EamArtifact.FILES_TYPE_ID) { + if (aType.getId() != CorrelationAttribute.FILES_TYPE_ID) { return false; } @@ -1521,7 +1496,7 @@ public abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ @Override - public void addReferenceInstance(EamGlobalFileInstance eamGlobalFileInstance, EamArtifact.Type correlationType) throws EamDbException { + public void addReferenceInstance(EamGlobalFileInstance eamGlobalFileInstance, CorrelationAttribute.Type correlationType) throws EamDbException { Connection conn = connect(); PreparedStatement preparedStatement = null; @@ -1549,7 +1524,7 @@ public abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ @Override - public void bulkInsertReferenceTypeEntries(Set globalInstances, EamArtifact.Type contentType) throws EamDbException { + public void bulkInsertReferenceTypeEntries(Set globalInstances, CorrelationAttribute.Type contentType) throws EamDbException { Connection conn = connect(); PreparedStatement bulkPs = null; @@ -1596,7 +1571,7 @@ public abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ @Override - public List getReferenceInstancesByTypeValue(EamArtifact.Type aType, String aValue) throws EamDbException { + public List getReferenceInstancesByTypeValue(CorrelationAttribute.Type aType, String aValue) throws EamDbException { Connection conn = connect(); List globalFileInstances = new ArrayList<>(); @@ -1632,7 +1607,7 @@ public abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ @Override - public int newCorrelationType(EamArtifact.Type newType) throws EamDbException { + public int newCorrelationType(CorrelationAttribute.Type newType) throws EamDbException { Connection conn = connect(); PreparedStatement preparedStatement = null; @@ -1673,7 +1648,7 @@ public abstract class AbstractSqlEamDb implements EamDb { resultSet = preparedStatementQuery.executeQuery(); if (resultSet.next()) { - EamArtifact.Type correlationType = getCorrelationTypeFromResultSet(resultSet); + CorrelationAttribute.Type correlationType = getCorrelationTypeFromResultSet(resultSet); typeId = correlationType.getId(); } } catch (SQLException ex) { @@ -1687,20 +1662,12 @@ public abstract class AbstractSqlEamDb implements EamDb { return typeId; } - /** - * Get the list of EamArtifact.Type's that will be used to correlate - * artifacts. - * - * @return List of EamArtifact.Type's. If none are defined in the database, - * the default list will be returned. - * - * @throws EamDbException - */ + @Override - public List getCorrelationTypes() throws EamDbException { + public List getDefinedCorrelationTypes() throws EamDbException { Connection conn = connect(); - List aTypes = new ArrayList<>(); + List aTypes = new ArrayList<>(); PreparedStatement preparedStatement = null; ResultSet resultSet = null; String sql = "SELECT * FROM correlation_types"; @@ -1732,10 +1699,10 @@ public abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ @Override - public List getEnabledCorrelationTypes() throws EamDbException { + public List getEnabledCorrelationTypes() throws EamDbException { Connection conn = connect(); - List aTypes = new ArrayList<>(); + List aTypes = new ArrayList<>(); PreparedStatement preparedStatement = null; ResultSet resultSet = null; String sql = "SELECT * FROM correlation_types WHERE enabled=1"; @@ -1767,10 +1734,10 @@ public abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ @Override - public List getSupportedCorrelationTypes() throws EamDbException { + public List getSupportedCorrelationTypes() throws EamDbException { Connection conn = connect(); - List aTypes = new ArrayList<>(); + List aTypes = new ArrayList<>(); PreparedStatement preparedStatement = null; ResultSet resultSet = null; String sql = "SELECT * FROM correlation_types WHERE supported=1"; @@ -1800,7 +1767,7 @@ public abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ @Override - public void updateCorrelationType(EamArtifact.Type aType) throws EamDbException { + public void updateCorrelationType(CorrelationAttribute.Type aType) throws EamDbException { Connection conn = connect(); PreparedStatement preparedStatement = null; @@ -1834,10 +1801,10 @@ public abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ @Override - public EamArtifact.Type getCorrelationTypeById(int typeId) throws EamDbException { + public CorrelationAttribute.Type getCorrelationTypeById(int typeId) throws EamDbException { Connection conn = connect(); - EamArtifact.Type aType; + CorrelationAttribute.Type aType; PreparedStatement preparedStatement = null; ResultSet resultSet = null; String sql = "SELECT * FROM correlation_types WHERE id=?"; @@ -1899,12 +1866,12 @@ public abstract class AbstractSqlEamDb implements EamDb { return eamCase; } - private EamDataSource getEamDataSourceFromResultSet(ResultSet resultSet) throws SQLException { + private CorrelationDataSource getEamDataSourceFromResultSet(ResultSet resultSet) throws SQLException { if (null == resultSet) { return null; } - EamDataSource eamDataSource = new EamDataSource( + CorrelationDataSource eamDataSource = new CorrelationDataSource( resultSet.getInt("id"), resultSet.getString("device_id"), resultSet.getString("name") @@ -1913,12 +1880,12 @@ public abstract class AbstractSqlEamDb implements EamDb { return eamDataSource; } - private EamArtifact.Type getCorrelationTypeFromResultSet(ResultSet resultSet) throws EamDbException, SQLException { + private CorrelationAttribute.Type getCorrelationTypeFromResultSet(ResultSet resultSet) throws EamDbException, SQLException { if (null == resultSet) { return null; } - EamArtifact.Type eamArtifactType = new EamArtifact.Type( + CorrelationAttribute.Type eamArtifactType = new CorrelationAttribute.Type( resultSet.getInt("id"), resultSet.getString("display_name"), resultSet.getString("db_table_name"), @@ -1939,17 +1906,17 @@ public abstract class AbstractSqlEamDb implements EamDb { * * @throws SQLException when an expected column name is not in the resultSet */ - private EamArtifactInstance getEamArtifactInstanceFromResultSet(ResultSet resultSet) throws SQLException { + private CorrelationAttributeInstance getEamArtifactInstanceFromResultSet(ResultSet resultSet) throws SQLException { if (null == resultSet) { return null; } - EamArtifactInstance eamArtifactInstance = new EamArtifactInstance( + CorrelationAttributeInstance eamArtifactInstance = new CorrelationAttributeInstance( new EamCase(resultSet.getString("case_uid"), resultSet.getString("case_name")), - new EamDataSource(resultSet.getString("device_id"), resultSet.getString("name")), + new CorrelationDataSource(-1, resultSet.getString("device_id"), resultSet.getString("name")), resultSet.getString("file_path"), resultSet.getString("comment"), TskData.FileKnown.valueOf(resultSet.getByte("known_status")), - EamArtifactInstance.GlobalStatus.LOCAL + CorrelationAttributeInstance.GlobalStatus.LOCAL ); return eamArtifactInstance; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifact.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttribute.java similarity index 85% rename from Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifact.java rename to Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttribute.java index 7c7a7b0afb..838aacfcbc 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifact.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttribute.java @@ -26,17 +26,18 @@ import java.util.regex.Pattern; import org.openide.util.NbBundle.Messages; /** - * - * Used to store info about a specific artifact. + * Represents a type and value pair that can be used for correlation. + * CorrelationAttributeInstances store information about the actual + * occurences of the attribute. */ -public class EamArtifact implements Serializable { +public class CorrelationAttribute implements Serializable { private static final long serialVersionUID = 1L; private String ID; private String correlationValue; private Type correlationType; - private final List artifactInstances; + private final List artifactInstances; // Type ID's for Default Correlation Types public static final int FILES_TYPE_ID = 0; @@ -55,17 +56,17 @@ public class EamArtifact implements Serializable { "CorrelationType.EMAIL.displayName=Email Addresses", "CorrelationType.PHONE.displayName=Phone Numbers", "CorrelationType.USBID.displayName=USB Devices"}) - public static List getDefaultCorrelationTypes() throws EamDbException { - List DEFAULT_CORRELATION_TYPES = new ArrayList<>(); - DEFAULT_CORRELATION_TYPES.add(new EamArtifact.Type(FILES_TYPE_ID, Bundle.CorrelationType_FILES_displayName(), "file", true, true)); // NON-NLS - DEFAULT_CORRELATION_TYPES.add(new EamArtifact.Type(DOMAIN_TYPE_ID, Bundle.CorrelationType_DOMAIN_displayName(), "domain", true, false)); // NON-NLS - DEFAULT_CORRELATION_TYPES.add(new EamArtifact.Type(EMAIL_TYPE_ID, Bundle.CorrelationType_EMAIL_displayName(), "email_address", true, false)); // NON-NLS - DEFAULT_CORRELATION_TYPES.add(new EamArtifact.Type(PHONE_TYPE_ID, Bundle.CorrelationType_PHONE_displayName(), "phone_number", true, false)); // NON-NLS - DEFAULT_CORRELATION_TYPES.add(new EamArtifact.Type(USBID_TYPE_ID, Bundle.CorrelationType_USBID_displayName(), "usb_devices", true, false)); // NON-NLS + public static List getDefaultCorrelationTypes() throws EamDbException { + List DEFAULT_CORRELATION_TYPES = new ArrayList<>(); + DEFAULT_CORRELATION_TYPES.add(new CorrelationAttribute.Type(FILES_TYPE_ID, Bundle.CorrelationType_FILES_displayName(), "file", true, true)); // NON-NLS + DEFAULT_CORRELATION_TYPES.add(new CorrelationAttribute.Type(DOMAIN_TYPE_ID, Bundle.CorrelationType_DOMAIN_displayName(), "domain", true, false)); // NON-NLS + DEFAULT_CORRELATION_TYPES.add(new CorrelationAttribute.Type(EMAIL_TYPE_ID, Bundle.CorrelationType_EMAIL_displayName(), "email_address", true, false)); // NON-NLS + DEFAULT_CORRELATION_TYPES.add(new CorrelationAttribute.Type(PHONE_TYPE_ID, Bundle.CorrelationType_PHONE_displayName(), "phone_number", true, false)); // NON-NLS + DEFAULT_CORRELATION_TYPES.add(new CorrelationAttribute.Type(USBID_TYPE_ID, Bundle.CorrelationType_USBID_displayName(), "usb_devices", true, false)); // NON-NLS return DEFAULT_CORRELATION_TYPES; } - public EamArtifact(Type correlationType, String correlationValue) { + public CorrelationAttribute(Type correlationType, String correlationValue) { this.ID = ""; this.correlationType = correlationType; // Lower-case all values to normalize and improve correlation hits, going forward make sure this makes sense for all correlation types @@ -73,7 +74,7 @@ public class EamArtifact implements Serializable { this.artifactInstances = new ArrayList<>(); } - public Boolean equals(EamArtifact otherArtifact) { + public Boolean equals(CorrelationAttribute otherArtifact) { return ((this.getID().equals(otherArtifact.getID())) && (this.getCorrelationType().equals(otherArtifact.getCorrelationType())) && (this.getCorrelationValue().equals(otherArtifact.getCorrelationValue())) @@ -82,6 +83,8 @@ public class EamArtifact implements Serializable { @Override public String toString() { + // NOTE: This string is currently being used in IngestEventsListener to detect if we have already seen + // the value and type pair. Be careful if this method is changed. String result = this.getID() + this.getCorrelationType().toString() + this.getCorrelationValue(); @@ -136,14 +139,14 @@ public class EamArtifact implements Serializable { * @return the List of artifactInstances; empty list of none have been * added. */ - public List getInstances() { + public List getInstances() { return new ArrayList<>(artifactInstances); } /** * @param artifactInstances the List of artifactInstances to set. */ - public void setInstances(List artifactInstances) { + public void setInstances(List artifactInstances) { this.artifactInstances.clear(); if (null != artifactInstances) { this.artifactInstances.addAll(artifactInstances); @@ -153,7 +156,7 @@ public class EamArtifact implements Serializable { /** * @param instance the instance to add */ - public void addInstance(EamArtifactInstance artifactInstance) { + public void addInstance(CorrelationAttributeInstance artifactInstance) { this.artifactInstances.add(artifactInstance); } @@ -214,10 +217,10 @@ public class EamArtifact implements Serializable { public boolean equals(Object that) { if (this == that) { return true; - } else if (!(that instanceof EamArtifact.Type)) { + } else if (!(that instanceof CorrelationAttribute.Type)) { return false; } else { - return ((EamArtifact.Type) that).sameType(this); + return ((CorrelationAttribute.Type) that).sameType(this); } } @@ -229,7 +232,7 @@ public class EamArtifact implements Serializable { * * @return true if it is the same type */ - private boolean sameType(EamArtifact.Type that) { + private boolean sameType(CorrelationAttribute.Type that) { return this.id == that.getId() && Objects.equals(this.supported, that.isSupported()) && Objects.equals(this.enabled, that.isEnabled()); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactInstance.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java similarity index 87% rename from Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactInstance.java rename to Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java index 4e73c7562b..e6f2aab4b2 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactInstance.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java @@ -24,7 +24,8 @@ import org.sleuthkit.datamodel.TskData; /** * - * Used to store info about a specific Artifact Instance. + * Used to store details about a specific instance of a + * CorrelationAttribute. Includes its data source, path, etc. * */ @Messages({"EamArtifactInstances.globalStatus.local=Local", @@ -32,7 +33,7 @@ import org.sleuthkit.datamodel.TskData; "EamArtifactInstances.knownStatus.bad=Bad", "EamArtifactInstances.knownStatus.known=Known", "EamArtifactInstances.knownStatus.unknown=Unknown"}) -public class EamArtifactInstance implements Serializable { +public class CorrelationAttributeInstance implements Serializable { public enum GlobalStatus { LOCAL(Bundle.EamArtifactInstances_globalStatus_local()), @@ -54,39 +55,39 @@ public class EamArtifactInstance implements Serializable { private String ID; private EamCase eamCase; - private EamDataSource eamDataSource; + private CorrelationDataSource eamDataSource; private String filePath; private String comment; private TskData.FileKnown knownStatus; private GlobalStatus globalStatus; - public EamArtifactInstance( + public CorrelationAttributeInstance( EamCase eamCase, - EamDataSource eamDataSource + CorrelationDataSource eamDataSource ) { this("", eamCase, eamDataSource, "", null, TskData.FileKnown.UNKNOWN, GlobalStatus.LOCAL); } - public EamArtifactInstance( + public CorrelationAttributeInstance( EamCase eamCase, - EamDataSource eamDataSource, + CorrelationDataSource eamDataSource, String filePath ) { this("", eamCase, eamDataSource, filePath, null, TskData.FileKnown.UNKNOWN, GlobalStatus.LOCAL); } - public EamArtifactInstance( + public CorrelationAttributeInstance( EamCase eamCase, - EamDataSource eamDataSource, + CorrelationDataSource eamDataSource, String filePath, String comment ) { this("", eamCase, eamDataSource, filePath, comment, TskData.FileKnown.UNKNOWN, GlobalStatus.LOCAL); } - public EamArtifactInstance( + public CorrelationAttributeInstance( EamCase eamCase, - EamDataSource eamDataSource, + CorrelationDataSource eamDataSource, String filePath, String comment, TskData.FileKnown knownStatus, @@ -95,10 +96,10 @@ public class EamArtifactInstance implements Serializable { this("", eamCase, eamDataSource, filePath, comment, knownStatus, globalStatus); } - public EamArtifactInstance( + public CorrelationAttributeInstance( String ID, EamCase eamCase, - EamDataSource eamDataSource, + CorrelationDataSource eamDataSource, String filePath, String comment, TskData.FileKnown knownStatus, @@ -114,7 +115,7 @@ public class EamArtifactInstance implements Serializable { this.globalStatus = globalStatus; } - public Boolean equals(EamArtifactInstance otherInstance) { + public Boolean equals(CorrelationAttributeInstance otherInstance) { return ((this.getID().equals(otherInstance.getID())) && (this.getEamCase().equals(otherInstance.getEamCase())) && (this.getEamDataSource().equals(otherInstance.getEamDataSource())) @@ -166,14 +167,14 @@ public class EamArtifactInstance implements Serializable { /** * @return the eamDataSource */ - public EamDataSource getEamDataSource() { + public CorrelationDataSource getEamDataSource() { return eamDataSource; } /** * @param eamDataSource the eamDataSource to set */ - public void setEamDataSource(EamDataSource eamDataSource) { + public void setEamDataSource(CorrelationDataSource eamDataSource) { this.eamDataSource = eamDataSource; } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java new file mode 100755 index 0000000000..9b7480bee7 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java @@ -0,0 +1,103 @@ +/* + * Central Repository + * + * Copyright 2015-2017 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.centralrepository.datamodel; + +import java.io.Serializable; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskDataException; + +/** + * + * Stores information about a Data Source in the correlation engine + * + */ +public class CorrelationDataSource implements Serializable { + + private static final long serialVersionUID = 1L; + + private final int dataSourceId; //< Id in the central repo + private final String deviceID; + private final String name; + + + CorrelationDataSource(int dataSourceId, + String deviceID, + String name) { + this.dataSourceId = dataSourceId; + this.deviceID = deviceID; + this.name = name; + } + + /** + * Create a CorrelationDataSource object from a TSK Content object. + * + * @param dataSource + * @return + * @throws EamDbException + */ + public static CorrelationDataSource fromTSKDataSource(Content dataSource) throws EamDbException { + Case curCase; + try { + curCase = Case.getCurrentCase(); + } catch (IllegalStateException ex) { + throw new EamDbException("Autopsy case is closed"); + } + String deviceId; + try { + deviceId = curCase.getSleuthkitCase().getDataSource(dataSource.getId()).getDeviceId(); + } catch (TskDataException | TskCoreException ex) { + throw new EamDbException("Error getting data source info: " + ex.getMessage()); + } + return new CorrelationDataSource(-1, deviceId, dataSource.getName()); + } + + @Override + public String toString() { + StringBuilder str = new StringBuilder(); + str.append("("); + str.append("ID=").append(Integer.toString(getDataSourceID())); + str.append(",deviceID=").append(getDeviceID()); + str.append(",name=").append(getName()); + str.append(")"); + return str.toString(); + } + + /** + * @return the ID + */ + public int getDataSourceID() { + return dataSourceId; + } + + /** + * @return the deviceID + */ + public String getDeviceID() { + return deviceID; + } + + /** + * @return the name + */ + public String getName() { + return name; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java index c8bfdddad7..7e3c7881be 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java @@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.centralrepository.datamodel; import java.util.ArrayList; import java.util.List; import java.util.logging.Level; +import org.openide.util.Exceptions; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; @@ -55,53 +56,65 @@ public class EamArtifactUtil { * null. * * @param bbArtifact BlackboardArtifact to examine + * @param addInstanceDetails If true, add instance details from bbArtifact into the returned structure + * @param checkEnabled If true, only create a CorrelationAttribute if it is enabled + * @return List of EamArtifacts */ - public static List fromBlackboardArtifact(BlackboardArtifact bbArtifact, - boolean includeInstances, - List artifactTypes, - boolean checkEnabled) { + public static List getCorrelationAttributeFromBlackboardArtifact(BlackboardArtifact bbArtifact, + boolean addInstanceDetails, boolean checkEnabled) { + + List eamArtifacts = new ArrayList<>(); - List eamArtifacts = new ArrayList<>(); - - for (EamArtifact.Type aType : artifactTypes) { - if ((checkEnabled && aType.isEnabled()) || !checkEnabled) { - EamArtifact eamArtifact = getTypeFromBlackboardArtifact(aType, bbArtifact); - if (eamArtifact != null) { - eamArtifacts.add(eamArtifact); + try { + // Cycle through the types and see if there is a correlation attribute that works + // for the given blackboard artifact + // + // @@@ This seems ineffecient. Instead of cycling based on correlation type, we should just + // have switch based on artifact type + for (CorrelationAttribute.Type aType : EamDb.getInstance().getDefinedCorrelationTypes()) { + if ((checkEnabled && aType.isEnabled()) || !checkEnabled) { + CorrelationAttribute eamArtifact = EamArtifactUtil.getCorrelationAttributeFromBlackboardArtifact(aType, bbArtifact); + if (eamArtifact != null) { + eamArtifacts.add(eamArtifact); + } } } + } catch (EamDbException ex) { + LOGGER.log(Level.SEVERE, "Error getting defined correlation types.", ex); // NON-NLS + return eamArtifacts; } - if (!eamArtifacts.isEmpty() && includeInstances) { + // if they asked for it, add the instance details associated with this occurance. + if (!eamArtifacts.isEmpty() && addInstanceDetails) { try { - AbstractFile af = Case.getCurrentCase().getSleuthkitCase().getAbstractFileById(bbArtifact.getObjectID()); - if (null == af) { - return null; + Case currentCase = Case.getCurrentCase(); + AbstractFile bbSourceFile = currentCase.getSleuthkitCase().getAbstractFileById(bbArtifact.getObjectID()); + if (null == bbSourceFile) { + //@@@ Log this + return eamArtifacts; } - String deviceId = ""; - try { - deviceId = Case.getCurrentCase().getSleuthkitCase().getDataSource(af.getDataSource().getId()).getDeviceId(); - } catch (TskCoreException | TskDataException ex) { - LOGGER.log(Level.SEVERE, "Error, failed to get deviceID or data source from current case.", ex); - } - - EamArtifactInstance eamInstance = new EamArtifactInstance( - new EamCase(Case.getCurrentCase().getName(), Case.getCurrentCase().getDisplayName()), - new EamDataSource(deviceId, af.getDataSource().getName()), - af.getParentPath() + af.getName(), + // make an instance for the BB source file + CorrelationAttributeInstance eamInstance = new CorrelationAttributeInstance( + new EamCase(currentCase.getName(), currentCase.getDisplayName()), + CorrelationDataSource.fromTSKDataSource(bbSourceFile.getDataSource()), + bbSourceFile.getParentPath() + bbSourceFile.getName(), "", TskData.FileKnown.UNKNOWN, - EamArtifactInstance.GlobalStatus.LOCAL + CorrelationAttributeInstance.GlobalStatus.LOCAL ); - for (EamArtifact eamArtifact : eamArtifacts) { + // add the instance details + for (CorrelationAttribute eamArtifact : eamArtifacts) { eamArtifact.addInstance(eamInstance); } - } catch (TskCoreException ex) { + } catch (TskCoreException | EamDbException ex) { LOGGER.log(Level.SEVERE, "Error creating artifact instance.", ex); // NON-NLS - return null; + return eamArtifacts; + } catch (IllegalStateException ex) { + LOGGER.log(Level.SEVERE, "Case is closed.", ex); // NON-NLS + return eamArtifacts; } } @@ -109,16 +122,16 @@ public class EamArtifactUtil { } /** - * Convert a blackboard artifact to an EamArtifact. - * Returns null if the converted artifact does not contain valid - * correlation data. + * Create an EamArtifact of type correlationType if one can be generated + * based on the data in the blackboard artifact. * - * @param aType The Central Repository artifact type to create - * @param bbArtifact The blackboard artifact to convert + * @param correlationType The Central Repository artifact type to create + * @param bbArtifact The blackboard artifact to pull data from * - * @return the new EamArtifact, or null if one was not created + * @return the new EamArtifact, or null if one was not created because bbArtifact + * did not contain the needed data */ - public static EamArtifact getTypeFromBlackboardArtifact(EamArtifact.Type aType, BlackboardArtifact bbArtifact) { + private static CorrelationAttribute getCorrelationAttributeFromBlackboardArtifact(CorrelationAttribute.Type correlationType, BlackboardArtifact bbArtifact) { String value = null; int artifactTypeID = bbArtifact.getArtifactTypeID(); @@ -128,10 +141,10 @@ public class EamArtifactUtil { BlackboardAttribute attribute = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT)); if (attribute != null) { BlackboardArtifact associatedArtifact = Case.getCurrentCase().getSleuthkitCase().getBlackboardArtifact(attribute.getValueLong()); - return getTypeFromBlackboardArtifact(aType, associatedArtifact); + return EamArtifactUtil.getCorrelationAttributeFromBlackboardArtifact(correlationType, associatedArtifact); } - } else if (aType.getId() == EamArtifact.EMAIL_TYPE_ID + } else if (correlationType.getId() == CorrelationAttribute.EMAIL_TYPE_ID && BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() == artifactTypeID) { BlackboardAttribute setNameAttr = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME)); @@ -139,7 +152,7 @@ public class EamArtifactUtil { && EamArtifactUtil.getEmailAddressAttrString().equals(setNameAttr.getValueString())) { value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD)).getValueString(); } - } else if (aType.getId() == EamArtifact.DOMAIN_TYPE_ID + } else if (correlationType.getId() == CorrelationAttribute.DOMAIN_TYPE_ID && (BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK.getTypeID() == artifactTypeID || BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE.getTypeID() == artifactTypeID || BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_DOWNLOAD.getTypeID() == artifactTypeID @@ -147,7 +160,7 @@ public class EamArtifactUtil { // Lower-case this to normalize domains value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN)).getValueString(); - } else if (aType.getId() == EamArtifact.PHONE_TYPE_ID + } else if (correlationType.getId() == CorrelationAttribute.PHONE_TYPE_ID && (BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT.getTypeID() == artifactTypeID || BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG.getTypeID() == artifactTypeID || BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE.getTypeID() == artifactTypeID)) { @@ -176,7 +189,7 @@ public class EamArtifactUtil { } } - } else if (aType.getId() == EamArtifact.USBID_TYPE_ID + } else if (correlationType.getId() == CorrelationAttribute.USBID_TYPE_ID && BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID() == artifactTypeID) { value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_ID)).getValueString(); @@ -188,7 +201,7 @@ public class EamArtifactUtil { } if (null != value) { - return new EamArtifact(aType, value); + return new CorrelationAttribute(correlationType, value); } else { return null; } @@ -196,15 +209,19 @@ public class EamArtifactUtil { /** * Create an EamArtifact from the given Content. - * Will return null if an artifact can not be created. Does not - * add the artifact to the database. + * Will return null if an artifact can not be created - this is not + * necessarily an error case, it just means an artifact can't be made. + * If creation fails due to an error (and not that the file is the wrong type + * or it has no hash), the error will be logged before returning. + * + * Does not add the artifact to the database. * * @param content The content object * @param knownStatus Unknown, notable, or known * @param comment The comment for the new artifact (generally used for a tag comment) * @return The new EamArtifact or null if creation failed */ - public static EamArtifact getEamArtifactFromContent(Content content, TskData.FileKnown knownStatus, String comment){ + public static CorrelationAttribute getEamArtifactFromContent(Content content, TskData.FileKnown knownStatus, String comment){ if(! (content instanceof AbstractFile)){ return null; @@ -220,45 +237,29 @@ public class EamArtifactUtil { || (!af.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC))) { return null; } - - String dsName; - try { - dsName = af.getDataSource().getName(); - } catch (TskCoreException ex) { - LOGGER.log(Level.SEVERE, "Error, unable to get name of data source from abstract file.", ex); - return null; - } // We need a hash to make the artifact String md5 = af.getMd5Hash(); if (md5 == null || md5.isEmpty()) { return null; } - - String deviceId; - try { - deviceId = Case.getCurrentCase().getSleuthkitCase().getDataSource(af.getDataSource().getId()).getDeviceId(); - } catch (TskCoreException | TskDataException ex) { - LOGGER.log(Level.SEVERE, "Error, failed to get deviceID or data source from current case.", ex); - return null; - } - EamArtifact eamArtifact; + CorrelationAttribute eamArtifact; try { - EamArtifact.Type filesType = EamDb.getInstance().getCorrelationTypeById(EamArtifact.FILES_TYPE_ID); - eamArtifact = new EamArtifact(filesType, af.getMd5Hash()); - EamArtifactInstance cei = new EamArtifactInstance( + CorrelationAttribute.Type filesType = EamDb.getInstance().getCorrelationTypeById(CorrelationAttribute.FILES_TYPE_ID); + eamArtifact = new CorrelationAttribute(filesType, af.getMd5Hash()); + CorrelationAttributeInstance cei = new CorrelationAttributeInstance( new EamCase(Case.getCurrentCase().getName(), Case.getCurrentCase().getDisplayName()), - new EamDataSource(deviceId, dsName), + CorrelationDataSource.fromTSKDataSource(af.getDataSource()), af.getParentPath() + af.getName(), comment, TskData.FileKnown.BAD, - EamArtifactInstance.GlobalStatus.LOCAL + CorrelationAttributeInstance.GlobalStatus.LOCAL ); eamArtifact.addInstance(cei); return eamArtifact; - } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error, unable to get FILES correlation type.", ex); + } catch (TskCoreException | EamDbException ex) { + LOGGER.log(Level.SEVERE, "Error making correlation attribute.", ex); return null; } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDataSource.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDataSource.java deleted file mode 100755 index cde6536e62..0000000000 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDataSource.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Central Repository - * - * Copyright 2015-2017 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.centralrepository.datamodel; - -import java.io.Serializable; - -/** - * - * Used to store info about a case. - * - */ -public class EamDataSource implements Serializable { - - private static long serialVersionUID = 1L; - - private int ID; - private String deviceID; - private String name; - - public EamDataSource(String deviceID, String name) { - this(-1, deviceID, name); - } - - public EamDataSource(int ID, - String deviceID, - String name) { - this.ID = ID; - this.deviceID = deviceID; - this.name = name; - } - - @Override - public String toString() { - StringBuilder str = new StringBuilder(); - str.append("("); - str.append("ID=").append(Integer.toString(getID())); - str.append(",deviceID=").append(getDeviceID()); - str.append(",name=").append(getName()); - str.append(")"); - return str.toString(); - } - - /** - * @return the ID - */ - public int getID() { - return ID; - } - - /** - * @param ID the ID to set - */ - public void setID(int ID) { - this.ID = ID; - } - - /** - * @return the deviceID - */ - public String getDeviceID() { - return deviceID; - } - - /** - * @param deviceID the deviceID to set - */ - public void setDeviceID(String deviceID) { - this.deviceID = deviceID; - } - - /** - * @return the name - */ - public String getName() { - return name; - } - - /** - * @param name the name to set - */ - public void setName(String name) { - this.name = name; - } - -} diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java index fff8bbf540..d3addef488 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java @@ -184,14 +184,14 @@ public interface EamDb { * * @param eamDataSource the data source to add */ - void newDataSource(EamDataSource eamDataSource) throws EamDbException; + void newDataSource(CorrelationDataSource eamDataSource) throws EamDbException; /** * Updates a Data Source in the database * * @param eamDataSource the data source to update */ - void updateDataSource(EamDataSource eamDataSource) throws EamDbException; + void updateDataSource(CorrelationDataSource eamDataSource) throws EamDbException; /** * Retrieves Data Source details based on data source device ID @@ -200,14 +200,14 @@ public interface EamDb { * * @return The data source */ - EamDataSource getDataSourceDetails(String dataSourceDeviceId) throws EamDbException; + CorrelationDataSource getDataSourceDetails(String dataSourceDeviceId) throws EamDbException; /** * Retrieves data sources that are in DB * * @return List of data sources */ - List getDataSources() throws EamDbException; + List getDataSources() throws EamDbException; /** * Inserts new Artifact(s) into the database. Should add associated Case and @@ -215,7 +215,7 @@ public interface EamDb { * * @param eamArtifact The artifact to add */ - void addArtifact(EamArtifact eamArtifact) throws EamDbException; + void addArtifact(CorrelationAttribute eamArtifact) throws EamDbException; /** * Retrieves eamArtifact instances from the database that are associated @@ -226,7 +226,7 @@ public interface EamDb { * * @return List of artifact instances for a given type/value */ - List getArtifactInstancesByTypeValue(EamArtifact.Type aType, String value) throws EamDbException; + List getArtifactInstancesByTypeValue(CorrelationAttribute.Type aType, String value) throws EamDbException; /** * Retrieves eamArtifact instances from the database that are associated @@ -239,7 +239,7 @@ public interface EamDb { * * @throws EamDbException */ - List getArtifactInstancesByPath(EamArtifact.Type aType, String filePath) throws EamDbException; + List getArtifactInstancesByPath(CorrelationAttribute.Type aType, String filePath) throws EamDbException; /** * Retrieves number of artifact instances in the database that are @@ -251,20 +251,16 @@ public interface EamDb { * @return Number of artifact instances having ArtifactType and * ArtifactValue. */ - Long getCountArtifactInstancesByTypeValue(EamArtifact.Type aType, String value) throws EamDbException; + Long getCountArtifactInstancesByTypeValue(CorrelationAttribute.Type aType, String value) throws EamDbException; /** - * Using the ArtifactType and ArtifactValue from the given eamArtfact, - * compute the ratio of: (The number of unique case_id/datasource_id tuples - * where Type/Value is found) divided by (The total number of unique - * case_id/datasource_id tuples in the database) expressed as a percentage. + * Calculate the percentage of data sources that have this attribute value. * - * @param aType EamArtifact.Type to search for - * @param value Value to search for + * @param corAttr Attribute type and value to get data about * * @return Int between 0 and 100 */ - int getCommonalityPercentageForTypeValue(EamArtifact.Type aType, String value) throws EamDbException; + int getFrequencyPercentage(CorrelationAttribute corAttr) throws EamDbException; /** * Retrieves number of unique caseDisplayName / dataSource tuples in the @@ -276,15 +272,14 @@ public interface EamDb { * * @return Number of unique tuples */ - Long getCountUniqueCaseDataSourceTuplesHavingTypeValue(EamArtifact.Type aType, String value) throws EamDbException; + Long getCountUniqueCaseDataSourceTuplesHavingTypeValue(CorrelationAttribute.Type aType, String value) throws EamDbException; /** - * Retrieves number of unique caseDisplayName/dataSource tuples in the - * database. + * Retrieves number of data sources in the database. * - * @return Number of unique tuples + * @return Number of unique data sources */ - Long getCountUniqueCaseDataSourceTuples() throws EamDbException; + Long getCountUniqueDataSources() throws EamDbException; /** * Retrieves number of eamArtifact instances in the database that are @@ -306,7 +301,7 @@ public interface EamDb { * * @param eamArtifact The artifact to add */ - void prepareBulkArtifact(EamArtifact eamArtifact) throws EamDbException; + void prepareBulkArtifact(CorrelationAttribute eamArtifact) throws EamDbException; /** * Executes a bulk insert of the eamArtifacts added from the @@ -326,7 +321,7 @@ public interface EamDb { * @param eamArtifact Artifact containing exactly one (1) ArtifactInstance. * @param FileKnown The status to change the artifact to */ - void setArtifactInstanceKnownStatus(EamArtifact eamArtifact, TskData.FileKnown knownStatus) throws EamDbException; + void setArtifactInstanceKnownStatus(CorrelationAttribute eamArtifact, TskData.FileKnown knownStatus) throws EamDbException; /** * Gets list of matching eamArtifact instances that have knownStatus = @@ -337,7 +332,7 @@ public interface EamDb { * * @return List with 0 or more matching eamArtifact instances. */ - List getArtifactInstancesKnownBad(EamArtifact.Type aType, String value) throws EamDbException; + List getArtifactInstancesKnownBad(CorrelationAttribute.Type aType, String value) throws EamDbException; /** * Count matching eamArtifacts instances that have knownStatus = "Bad". @@ -347,7 +342,7 @@ public interface EamDb { * * @return Number of matching eamArtifacts */ - Long getCountArtifactInstancesKnownBad(EamArtifact.Type aType, String value) throws EamDbException; + Long getCountArtifactInstancesKnownBad(CorrelationAttribute.Type aType, String value) throws EamDbException; /** * Gets list of distinct case display names, where each case has 1+ Artifact @@ -361,7 +356,7 @@ public interface EamDb { * * @throws EamDbException */ - List getListCasesHavingArtifactInstancesKnownBad(EamArtifact.Type aType, String value) throws EamDbException; + List getListCasesHavingArtifactInstancesKnownBad(CorrelationAttribute.Type aType, String value) throws EamDbException; /** * Is the artifact known as bad according to the reference entries? @@ -371,7 +366,7 @@ public interface EamDb { * * @return Global known status of the artifact */ - boolean isArtifactlKnownBadByReference(EamArtifact.Type aType, String value) throws EamDbException; + boolean isArtifactlKnownBadByReference(CorrelationAttribute.Type aType, String value) throws EamDbException; /** * Add a new organization @@ -433,7 +428,7 @@ public interface EamDb { * * @throws EamDbException */ - void addReferenceInstance(EamGlobalFileInstance eamGlobalFileInstance, EamArtifact.Type correlationType) throws EamDbException; + void addReferenceInstance(EamGlobalFileInstance eamGlobalFileInstance, CorrelationAttribute.Type correlationType) throws EamDbException; /** * Add a new global file instance to the bulk collection @@ -452,7 +447,7 @@ public interface EamDb { * * @throws EamDbException */ - void bulkInsertReferenceTypeEntries(Set globalInstances, EamArtifact.Type contentType) throws EamDbException; + void bulkInsertReferenceTypeEntries(Set globalInstances, CorrelationAttribute.Type contentType) throws EamDbException; /** * Get all reference entries having a given correlation type and value @@ -464,7 +459,7 @@ public interface EamDb { * * @throws EamDbException */ - List getReferenceInstancesByTypeValue(EamArtifact.Type aType, String aValue) throws EamDbException; + List getReferenceInstancesByTypeValue(CorrelationAttribute.Type aType, String aValue) throws EamDbException; /** * Add a new EamArtifact.Type to the db. @@ -475,18 +470,18 @@ public interface EamDb { * * @throws EamDbException */ - public int newCorrelationType(EamArtifact.Type newType) throws EamDbException; + public int newCorrelationType(CorrelationAttribute.Type newType) throws EamDbException; /** - * Get the list of EamArtifact.Type's that will be used to correlate - * artifacts. + * Get the list of EamArtifact.Type's that are defined in the DB and can be + * used to correlate artifacts. * * @return List of EamArtifact.Type's. If none are defined in the database, * the default list will be returned. * * @throws EamDbException */ - public List getCorrelationTypes() throws EamDbException; + public List getDefinedCorrelationTypes() throws EamDbException; /** * Get the list of enabled EamArtifact.Type's that will be used to correlate @@ -497,7 +492,7 @@ public interface EamDb { * * @throws EamDbException */ - public List getEnabledCorrelationTypes() throws EamDbException; + public List getEnabledCorrelationTypes() throws EamDbException; /** * Get the list of supported EamArtifact.Type's that can be used to @@ -508,7 +503,7 @@ public interface EamDb { * * @throws EamDbException */ - public List getSupportedCorrelationTypes() throws EamDbException; + public List getSupportedCorrelationTypes() throws EamDbException; /** * Update a EamArtifact.Type. @@ -517,7 +512,7 @@ public interface EamDb { * * @throws EamDbException */ - public void updateCorrelationType(EamArtifact.Type aType) throws EamDbException; + public void updateCorrelationType(CorrelationAttribute.Type aType) throws EamDbException; /** * Get the EamArtifact.Type that has the given Type.Id. @@ -528,5 +523,5 @@ public interface EamDb { * * @throws EamDbException */ - public EamArtifact.Type getCorrelationTypeById(int typeId) throws EamDbException; + public CorrelationAttribute.Type getCorrelationTypeById(int typeId) throws EamDbException; } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java index dc43b8ba51..5d70d6fdac 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java @@ -101,9 +101,9 @@ public class EamDbUtil { String sql = "INSERT INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?)"; try { - List DEFAULT_CORRELATION_TYPES = EamArtifact.getDefaultCorrelationTypes(); + List DEFAULT_CORRELATION_TYPES = CorrelationAttribute.getDefaultCorrelationTypes(); preparedStatement = conn.prepareStatement(sql); - for (EamArtifact.Type newType : DEFAULT_CORRELATION_TYPES) { + for (CorrelationAttribute.Type newType : DEFAULT_CORRELATION_TYPES) { preparedStatement.setInt(1, newType.getId()); preparedStatement.setString(2, newType.getDisplayName()); preparedStatement.setString(3, newType.getDbTableName()); @@ -230,7 +230,7 @@ public class EamDbUtil { * * @return Instance table name for this Type. */ - public static String correlationTypeToInstanceTableName(EamArtifact.Type type) { + public static String correlationTypeToInstanceTableName(CorrelationAttribute.Type type) { return type.getDbTableName() + "_instances"; } @@ -241,7 +241,7 @@ public class EamDbUtil { * * @return Reference table name for this Type. */ - public static String correlationTypeToReferenceTableName(EamArtifact.Type type) { + public static String correlationTypeToReferenceTableName(CorrelationAttribute.Type type) { return "reference_" + type.getDbTableName(); } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java index 208ded1998..cecba78f4e 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java @@ -112,10 +112,10 @@ public class PostgresEamDb extends AbstractSqlEamDb { String instancesTemplate = "TRUNCATE TABLE %s_instances RESTART IDENTITY CASCADE"; String referencesTemplate = "TRUNCATE TABLE reference_%s RESTART IDENTITY CASCADE"; - for (EamArtifact.Type type : DEFAULT_CORRELATION_TYPES) { + for (CorrelationAttribute.Type type : DEFAULT_CORRELATION_TYPES) { dropContent.executeUpdate(String.format(instancesTemplate, type.getDbTableName())); // FUTURE: support other reference types - if (type.getId() == EamArtifact.FILES_TYPE_ID) { + if (type.getId() == CorrelationAttribute.FILES_TYPE_ID) { dropContent.executeUpdate(String.format(referencesTemplate, type.getDbTableName())); } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java index 81a1aadd5e..6856b407f6 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java @@ -455,11 +455,11 @@ public final class PostgresEamDbSettings { stmt.execute(createDbInfoTable.toString()); // Create a separate instance and reference table for each correlation type - List DEFAULT_CORRELATION_TYPES = EamArtifact.getDefaultCorrelationTypes(); + List DEFAULT_CORRELATION_TYPES = CorrelationAttribute.getDefaultCorrelationTypes(); String reference_type_dbname; String instance_type_dbname; - for (EamArtifact.Type type : DEFAULT_CORRELATION_TYPES) { + for (CorrelationAttribute.Type type : DEFAULT_CORRELATION_TYPES) { reference_type_dbname = EamDbUtil.correlationTypeToReferenceTableName(type); instance_type_dbname = EamDbUtil.correlationTypeToInstanceTableName(type); @@ -470,7 +470,7 @@ public final class PostgresEamDbSettings { stmt.execute(String.format(instancesIdx4, instance_type_dbname, instance_type_dbname)); // FUTURE: allow more than the FILES type - if (type.getId() == EamArtifact.FILES_TYPE_ID) { + if (type.getId() == CorrelationAttribute.FILES_TYPE_ID) { stmt.execute(String.format(createReferenceTypesTableTemplate.toString(), reference_type_dbname, reference_type_dbname)); stmt.execute(String.format(referenceTypesIdx1, reference_type_dbname, reference_type_dbname)); stmt.execute(String.format(referenceTypesIdx2, reference_type_dbname, reference_type_dbname)); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java index 1e1bc7227e..536fe5e5aa 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java @@ -123,10 +123,10 @@ public class SqliteEamDb extends AbstractSqlEamDb { String instancesTemplate = "DELETE FROM %s_instances"; String referencesTemplate = "DELETE FROM global_files"; - for (EamArtifact.Type type : DEFAULT_CORRELATION_TYPES) { + for (CorrelationAttribute.Type type : DEFAULT_CORRELATION_TYPES) { dropContent.executeUpdate(String.format(instancesTemplate, type.getDbTableName())); // FUTURE: support other reference types - if (type.getId() == EamArtifact.FILES_TYPE_ID) { + if (type.getId() == CorrelationAttribute.FILES_TYPE_ID) { dropContent.executeUpdate(String.format(referencesTemplate, type.getDbTableName())); } } @@ -350,7 +350,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @param eamDataSource the data source to add */ @Override - public void newDataSource(EamDataSource eamDataSource) throws EamDbException { + public void newDataSource(CorrelationDataSource eamDataSource) throws EamDbException { try{ acquireExclusiveLock(); super.newDataSource(eamDataSource); @@ -365,7 +365,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @param eamDataSource the data source to update */ @Override - public void updateDataSource(EamDataSource eamDataSource) throws EamDbException { + public void updateDataSource(CorrelationDataSource eamDataSource) throws EamDbException { try{ acquireExclusiveLock(); super.updateDataSource(eamDataSource); @@ -382,7 +382,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @return The data source */ @Override - public EamDataSource getDataSourceDetails(String dataSourceDeviceId) throws EamDbException { + public CorrelationDataSource getDataSourceDetails(String dataSourceDeviceId) throws EamDbException { try{ acquireSharedLock(); return super.getDataSourceDetails(dataSourceDeviceId); @@ -397,7 +397,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @return list of data sources in the DB */ @Override - public List getDataSources() throws EamDbException { + public List getDataSources() throws EamDbException { try{ acquireSharedLock(); return super.getDataSources(); @@ -413,7 +413,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @param eamArtifact The artifact to add */ @Override - public void addArtifact(EamArtifact eamArtifact) throws EamDbException { + public void addArtifact(CorrelationAttribute eamArtifact) throws EamDbException { try{ acquireExclusiveLock(); super.addArtifact(eamArtifact); @@ -431,7 +431,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @return List of artifact instances for a given type/value */ @Override - public List getArtifactInstancesByTypeValue(EamArtifact.Type aType, String value) throws EamDbException { + public List getArtifactInstancesByTypeValue(CorrelationAttribute.Type aType, String value) throws EamDbException { try{ acquireSharedLock(); return super.getArtifactInstancesByTypeValue(aType, value); @@ -452,7 +452,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @throws EamDbException */ @Override - public List getArtifactInstancesByPath(EamArtifact.Type aType, String filePath) throws EamDbException { + public List getArtifactInstancesByPath(CorrelationAttribute.Type aType, String filePath) throws EamDbException { try{ acquireSharedLock(); return super.getArtifactInstancesByPath(aType, filePath); @@ -472,7 +472,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * ArtifactValue. */ @Override - public Long getCountArtifactInstancesByTypeValue(EamArtifact.Type aType, String value) throws EamDbException { + public Long getCountArtifactInstancesByTypeValue(CorrelationAttribute.Type aType, String value) throws EamDbException { try{ acquireSharedLock(); return super.getCountArtifactInstancesByTypeValue(aType, value); @@ -481,22 +481,11 @@ public class SqliteEamDb extends AbstractSqlEamDb { } } - /** - * Using the ArtifactType and ArtifactValue from the given eamArtfact, - * compute the ratio of: (The number of unique case_id/datasource_id tuples - * where Type/Value is found) divided by (The total number of unique - * case_id/datasource_id tuples in the database) expressed as a percentage. - * - * @param eamArtifact Artifact with artifactType and artifactValue to search - * for - * - * @return Int between 0 and 100 - */ @Override - public int getCommonalityPercentageForTypeValue(EamArtifact.Type aType, String value) throws EamDbException { + public int getFrequencyPercentage(CorrelationAttribute corAttr) throws EamDbException { try{ acquireSharedLock(); - return super.getCommonalityPercentageForTypeValue(aType, value); + return super.getFrequencyPercentage(corAttr); } finally { releaseSharedLock(); } @@ -513,7 +502,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @return Number of unique tuples */ @Override - public Long getCountUniqueCaseDataSourceTuplesHavingTypeValue(EamArtifact.Type aType, String value) throws EamDbException { + public Long getCountUniqueCaseDataSourceTuplesHavingTypeValue(CorrelationAttribute.Type aType, String value) throws EamDbException { try{ acquireSharedLock(); return super.getCountUniqueCaseDataSourceTuplesHavingTypeValue(aType, value); @@ -522,17 +511,12 @@ public class SqliteEamDb extends AbstractSqlEamDb { } } - /** - * Retrieves number of unique caseDisplayName/dataSource tuples in the - * database. - * - * @return Number of unique tuples - */ + @Override - public Long getCountUniqueCaseDataSourceTuples() throws EamDbException { + public Long getCountUniqueDataSources() throws EamDbException { try{ acquireSharedLock(); - return super.getCountUniqueCaseDataSourceTuples(); + return super.getCountUniqueDataSources(); } finally { releaseSharedLock(); } @@ -593,7 +577,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @param eamArtifact Artifact containing exactly one (1) ArtifactInstance. */ @Override - public void setArtifactInstanceKnownStatus(EamArtifact eamArtifact, TskData.FileKnown knownStatus) throws EamDbException { + public void setArtifactInstanceKnownStatus(CorrelationAttribute eamArtifact, TskData.FileKnown knownStatus) throws EamDbException { try{ acquireExclusiveLock(); super.setArtifactInstanceKnownStatus(eamArtifact, knownStatus); @@ -612,7 +596,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @return List with 0 or more matching eamArtifact instances. */ @Override - public List getArtifactInstancesKnownBad(EamArtifact.Type aType, String value) throws EamDbException { + public List getArtifactInstancesKnownBad(CorrelationAttribute.Type aType, String value) throws EamDbException { try{ acquireSharedLock(); return super.getArtifactInstancesKnownBad(aType, value); @@ -630,7 +614,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @return Number of matching eamArtifacts */ @Override - public Long getCountArtifactInstancesKnownBad(EamArtifact.Type aType, String value) throws EamDbException { + public Long getCountArtifactInstancesKnownBad(CorrelationAttribute.Type aType, String value) throws EamDbException { try{ acquireSharedLock(); return super.getCountArtifactInstancesKnownBad(aType, value); @@ -652,7 +636,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @throws EamDbException */ @Override - public List getListCasesHavingArtifactInstancesKnownBad(EamArtifact.Type aType, String value) throws EamDbException { + public List getListCasesHavingArtifactInstancesKnownBad(CorrelationAttribute.Type aType, String value) throws EamDbException { try{ acquireSharedLock(); return super.getListCasesHavingArtifactInstancesKnownBad(aType, value); @@ -670,7 +654,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @return Global known status of the artifact */ @Override - public boolean isArtifactlKnownBadByReference(EamArtifact.Type aType, String value) throws EamDbException { + public boolean isArtifactlKnownBadByReference(CorrelationAttribute.Type aType, String value) throws EamDbException { try{ acquireSharedLock(); return super.isArtifactlKnownBadByReference(aType, value); @@ -780,7 +764,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @throws EamDbException */ @Override - public void addReferenceInstance(EamGlobalFileInstance eamGlobalFileInstance, EamArtifact.Type correlationType) throws EamDbException { + public void addReferenceInstance(EamGlobalFileInstance eamGlobalFileInstance, CorrelationAttribute.Type correlationType) throws EamDbException { try{ acquireExclusiveLock(); super.addReferenceInstance(eamGlobalFileInstance, correlationType); @@ -795,7 +779,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @throws EamDbException */ @Override - public void bulkInsertReferenceTypeEntries(Set globalInstances, EamArtifact.Type contentType) throws EamDbException { + public void bulkInsertReferenceTypeEntries(Set globalInstances, CorrelationAttribute.Type contentType) throws EamDbException { try{ acquireExclusiveLock(); super.bulkInsertReferenceTypeEntries(globalInstances, contentType); @@ -815,7 +799,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @throws EamDbException */ @Override - public List getReferenceInstancesByTypeValue(EamArtifact.Type aType, String aValue) throws EamDbException { + public List getReferenceInstancesByTypeValue(CorrelationAttribute.Type aType, String aValue) throws EamDbException { try{ acquireSharedLock(); return super.getReferenceInstancesByTypeValue(aType, aValue); @@ -834,7 +818,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @throws EamDbException */ @Override - public int newCorrelationType(EamArtifact.Type newType) throws EamDbException { + public int newCorrelationType(CorrelationAttribute.Type newType) throws EamDbException { try{ acquireExclusiveLock(); return super.newCorrelationType(newType); @@ -853,10 +837,10 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @throws EamDbException */ @Override - public List getCorrelationTypes() throws EamDbException { + public List getDefinedCorrelationTypes() throws EamDbException { try{ acquireSharedLock(); - return super.getCorrelationTypes(); + return super.getDefinedCorrelationTypes(); } finally { releaseSharedLock(); } @@ -872,7 +856,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @throws EamDbException */ @Override - public List getEnabledCorrelationTypes() throws EamDbException { + public List getEnabledCorrelationTypes() throws EamDbException { try{ acquireSharedLock(); return super.getEnabledCorrelationTypes(); @@ -891,7 +875,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @throws EamDbException */ @Override - public List getSupportedCorrelationTypes() throws EamDbException { + public List getSupportedCorrelationTypes() throws EamDbException { try{ acquireSharedLock(); return super.getSupportedCorrelationTypes(); @@ -908,7 +892,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @throws EamDbException */ @Override - public void updateCorrelationType(EamArtifact.Type aType) throws EamDbException { + public void updateCorrelationType(CorrelationAttribute.Type aType) throws EamDbException { try{ acquireExclusiveLock(); super.updateCorrelationType(aType); @@ -927,7 +911,7 @@ public class SqliteEamDb extends AbstractSqlEamDb { * @throws EamDbException */ @Override - public EamArtifact.Type getCorrelationTypeById(int typeId) throws EamDbException { + public CorrelationAttribute.Type getCorrelationTypeById(int typeId) throws EamDbException { try{ acquireSharedLock(); return super.getCorrelationTypeById(typeId); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java index ca3dec0c33..1010b6b56e 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java @@ -403,11 +403,11 @@ public final class SqliteEamDbSettings { stmt.execute(createDbInfoTable.toString()); // Create a separate instance and reference table for each artifact type - List DEFAULT_CORRELATION_TYPES = EamArtifact.getDefaultCorrelationTypes(); + List DEFAULT_CORRELATION_TYPES = CorrelationAttribute.getDefaultCorrelationTypes(); String reference_type_dbname; String instance_type_dbname; - for (EamArtifact.Type type : DEFAULT_CORRELATION_TYPES) { + for (CorrelationAttribute.Type type : DEFAULT_CORRELATION_TYPES) { reference_type_dbname = EamDbUtil.correlationTypeToReferenceTableName(type); instance_type_dbname = EamDbUtil.correlationTypeToInstanceTableName(type); @@ -418,7 +418,7 @@ public final class SqliteEamDbSettings { stmt.execute(String.format(instancesIdx4, instance_type_dbname, instance_type_dbname)); // FUTURE: allow more than the FILES type - if (type.getId() == EamArtifact.FILES_TYPE_ID) { + if (type.getId() == CorrelationAttribute.FILES_TYPE_ID) { stmt.execute(String.format(createReferenceTypesTableTemplate.toString(), reference_type_dbname, reference_type_dbname)); stmt.execute(String.format(referenceTypesIdx1, reference_type_dbname, reference_type_dbname)); stmt.execute(String.format(referenceTypesIdx2, reference_type_dbname, reference_type_dbname)); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java index 6a38a7e7b8..438f1f31fe 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java @@ -32,10 +32,10 @@ import org.sleuthkit.autopsy.casemodule.events.ContentTagDeletedEvent; import org.sleuthkit.autopsy.casemodule.events.DataSourceAddedEvent; import org.sleuthkit.autopsy.casemodule.services.TagsManager; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifact; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttribute; import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil; import org.sleuthkit.autopsy.centralrepository.datamodel.EamCase; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDataSource; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationDataSource; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import org.sleuthkit.autopsy.centralrepository.datamodel.EamOrganization; @@ -139,14 +139,16 @@ public class CaseEventListener implements PropertyChangeListener { } } - final EamArtifact eamArtifact = EamArtifactUtil.getEamArtifactFromContent(af, + final CorrelationAttribute eamArtifact = EamArtifactUtil.getEamArtifactFromContent(af, knownStatus, comment); - // send update to Central Repository db - Runnable r = new KnownStatusChangeRunner(eamArtifact, knownStatus); - // TODO: send r into a thread pool instead - Thread t = new Thread(r); - t.start(); + if(eamArtifact != null){ + // send update to Central Repository db + Runnable r = new KnownStatusChangeRunner(eamArtifact, knownStatus); + // TODO: send r into a thread pool instead + Thread t = new Thread(r); + t.start(); + } } // CONTENT_TAG_ADDED, CONTENT_TAG_DELETED break; @@ -220,18 +222,15 @@ public class CaseEventListener implements PropertyChangeListener { return; } - try { - List convertedArtifacts = EamArtifactUtil.fromBlackboardArtifact(bbArtifact, true, dbManager.getCorrelationTypes(), true); - for (EamArtifact eamArtifact : convertedArtifacts) { - eamArtifact.getInstances().get(0).setComment(comment); - Runnable r = new KnownStatusChangeRunner(eamArtifact, knownStatus); - // TODO: send r into a thread pool instead - Thread t = new Thread(r); - t.start(); - } - } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error, unable to get artifact types during BLACKBOARD_ARTIFACT_TAG_ADDED/BLACKBOARD_ARTIFACT_TAG_DELETED event.", ex); + List convertedArtifacts = EamArtifactUtil.getCorrelationAttributeFromBlackboardArtifact(bbArtifact, true, true); + for (CorrelationAttribute eamArtifact : convertedArtifacts) { + eamArtifact.getInstances().get(0).setComment(comment); + Runnable r = new KnownStatusChangeRunner(eamArtifact, knownStatus); + // TODO: send r into a thread pool instead + Thread t = new Thread(r); + t.start(); } + } // BLACKBOARD_ARTIFACT_TAG_ADDED, BLACKBOARD_ARTIFACT_TAG_DELETED break; @@ -245,9 +244,8 @@ public class CaseEventListener implements PropertyChangeListener { try { String deviceId = Case.getCurrentCase().getSleuthkitCase().getDataSource(newDataSource.getId()).getDeviceId(); - if (null == dbManager.getDataSourceDetails(deviceId)) { - dbManager.newDataSource(new EamDataSource(deviceId, newDataSource.getName())); + dbManager.newDataSource(CorrelationDataSource.fromTSKDataSource(newDataSource)); } } catch (EamDbException ex) { LOGGER.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); //NON-NLS diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index cdedd03c83..3f9e3e14aa 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -34,7 +34,7 @@ import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifact; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttribute; import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import org.sleuthkit.datamodel.AbstractFile; @@ -49,7 +49,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; */ public class IngestEventsListener { - private static final Logger LOGGER = Logger.getLogger(EamArtifact.class.getName()); + private static final Logger LOGGER = Logger.getLogger(CorrelationAttribute.class.getName()); final Collection addedCeArtifactTrackerSet = new LinkedHashSet<>(); private static int ceModuleInstanceCount = 0; @@ -131,34 +131,32 @@ public class IngestEventsListener { if (null == bbArtifacts) { //the ModuleDataEvents don't always have a collection of artifacts set return; } - List eamArtifacts = new ArrayList<>(); - try { - for (BlackboardArtifact bbArtifact : bbArtifacts) { - // eamArtifact will be null OR a EamArtifact containing one EamArtifactInstance. - List convertedArtifacts = EamArtifactUtil.fromBlackboardArtifact(bbArtifact, true, dbManager.getCorrelationTypes(), true); - for (EamArtifact eamArtifact : convertedArtifacts) { - try { - // Only do something with this artifact if it's unique within the job - if (addedCeArtifactTrackerSet.add(eamArtifact.toString())) { - // Was it previously marked as bad? - // query db for artifact instances having this TYPE/VALUE and knownStatus = "Bad". - // if gettKnownStatus() is "Unknown" and this artifact instance was marked bad in a previous case, - // create TSK_INTERESTING_ARTIFACT_HIT artifact on BB. - List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue()); - if (!caseDisplayNames.isEmpty()) { - postCorrelatedBadArtifactToBlackboard(bbArtifact, - caseDisplayNames); - } - eamArtifacts.add(eamArtifact); + List eamArtifacts = new ArrayList<>(); + + for (BlackboardArtifact bbArtifact : bbArtifacts) { + // eamArtifact will be null OR a EamArtifact containing one EamArtifactInstance. + List convertedArtifacts = EamArtifactUtil.getCorrelationAttributeFromBlackboardArtifact(bbArtifact, true, true); + for (CorrelationAttribute eamArtifact : convertedArtifacts) { + try { + // Only do something with this artifact if it's unique within the job + if (addedCeArtifactTrackerSet.add(eamArtifact.toString())) { + // Was it previously marked as bad? + // query db for artifact instances having this TYPE/VALUE and knownStatus = "Bad". + // if gettKnownStatus() is "Unknown" and this artifact instance was marked bad in a previous case, + // create TSK_INTERESTING_ARTIFACT_HIT artifact on BB. + List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue()); + if (!caseDisplayNames.isEmpty()) { + postCorrelatedBadArtifactToBlackboard(bbArtifact, + caseDisplayNames); } - } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error counting notable artifacts.", ex); + eamArtifacts.add(eamArtifact); } + } catch (EamDbException ex) { + LOGGER.log(Level.SEVERE, "Error counting notable artifacts.", ex); } } - } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error getting correlation types.", ex); } + if (FALSE == eamArtifacts.isEmpty()) { // send update to entperirse artifact manager db Runnable r = new NewArtifactsRunner(eamArtifacts); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/KnownStatusChangeRunner.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/KnownStatusChangeRunner.java index 8e3e0c496e..4761f77370 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/KnownStatusChangeRunner.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/KnownStatusChangeRunner.java @@ -20,7 +20,7 @@ package org.sleuthkit.autopsy.centralrepository.eventlisteners; import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifact; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttribute; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.datamodel.TskData.FileKnown; @@ -33,10 +33,10 @@ public class KnownStatusChangeRunner implements Runnable { private static final Logger LOGGER = Logger.getLogger(KnownStatusChangeRunner.class.getName()); private static final long serialVersionUID = 1L; - private final EamArtifact artifact; + private final CorrelationAttribute artifact; private final FileKnown knownStatus; - public KnownStatusChangeRunner(EamArtifact artifact, FileKnown knownStatus) { + public KnownStatusChangeRunner(CorrelationAttribute artifact, FileKnown knownStatus) { this.artifact = artifact; this.knownStatus = knownStatus; } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/NewArtifactsRunner.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/NewArtifactsRunner.java index 1aabf9bd48..11697e7215 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/NewArtifactsRunner.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/NewArtifactsRunner.java @@ -22,7 +22,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifact; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttribute; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; @@ -34,10 +34,10 @@ public class NewArtifactsRunner implements Runnable { private static final Logger LOGGER = Logger.getLogger(NewArtifactsRunner.class.getName()); private static final long serialVersionUID = 1L; - private final Collection eamArtifacts; + private final Collection eamArtifacts; @SuppressWarnings(value = {"unchecked", "rawtypes"}) - public NewArtifactsRunner(Collection eamArtifacts) { + public NewArtifactsRunner(Collection eamArtifacts) { this.eamArtifacts = new ArrayList(eamArtifacts); } @@ -51,7 +51,7 @@ public class NewArtifactsRunner implements Runnable { try { EamDb dbManager = EamDb.getInstance(); - for (EamArtifact eamArtifact : eamArtifacts) { + for (CorrelationAttribute eamArtifact : eamArtifacts) { dbManager.addArtifact(eamArtifact); } } catch (EamDbException ex) { diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 0b283d9ec1..518ac55a1a 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -23,6 +23,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import java.util.List; import java.util.logging.Level; import java.util.stream.Collectors; +import org.openide.util.Exceptions; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.casemodule.Case; @@ -35,9 +36,9 @@ import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifact; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactInstance; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDataSource; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttribute; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationDataSource; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbPlatformEnum; import org.sleuthkit.datamodel.AbstractFile; @@ -64,9 +65,9 @@ class IngestModule implements FileIngestModule { private static final IngestModuleReferenceCounter warningMsgRefCounter = new IngestModuleReferenceCounter(); private long jobId; private EamCase eamCase; - private EamDataSource eamDataSource; + private CorrelationDataSource eamDataSource; private Blackboard blackboard; - private EamArtifact.Type filesType; + private CorrelationAttribute.Type filesType; @Override public ProcessResult process(AbstractFile af) { @@ -137,14 +138,14 @@ class IngestModule implements FileIngestModule { } try { - EamArtifact eamArtifact = new EamArtifact(filesType, md5); - EamArtifactInstance cefi = new EamArtifactInstance( + CorrelationAttribute eamArtifact = new CorrelationAttribute(filesType, md5); + CorrelationAttributeInstance cefi = new CorrelationAttributeInstance( eamCase, eamDataSource, af.getParentPath() + af.getName(), null, TskData.FileKnown.UNKNOWN, - EamArtifactInstance.GlobalStatus.LOCAL + CorrelationAttributeInstance.GlobalStatus.LOCAL ); eamArtifact.addInstance(cefi); dbManager.prepareBulkArtifact(eamArtifact); @@ -217,16 +218,13 @@ class IngestModule implements FileIngestModule { jobId = context.getJobId(); eamCase = new EamCase(Case.getCurrentCase().getName(), Case.getCurrentCase().getDisplayName()); - String deviceId; try { - deviceId = Case.getCurrentCase().getSleuthkitCase().getDataSource(context.getDataSource().getId()).getDeviceId(); - } catch (TskCoreException | TskDataException ex) { - LOGGER.log(Level.SEVERE, "Error getting data source device id in ingest module start up.", ex); // NON-NLS - throw new IngestModuleException("Error getting data source device id in ingest module start up.", ex); // NON-NLS + eamDataSource = CorrelationDataSource.fromTSKDataSource(context.getDataSource()); + } catch (EamDbException ex) { + LOGGER.log(Level.SEVERE, "Error getting data source info.", ex); // NON-NLS + throw new IngestModuleException("Error getting data source info.", ex); // NON-NLS } - eamDataSource = new EamDataSource(deviceId, context.getDataSource().getName()); - EamDb dbManager; try { dbManager = EamDb.getInstance(); @@ -236,7 +234,7 @@ class IngestModule implements FileIngestModule { } try { - filesType = dbManager.getCorrelationTypeById(EamArtifact.FILES_TYPE_ID); + filesType = dbManager.getCorrelationTypeById(CorrelationAttribute.FILES_TYPE_ID); } catch (EamDbException ex) { LOGGER.log(Level.SEVERE, "Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS throw new IngestModuleException("Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/license-centralrepository.txt b/Core/src/org/sleuthkit/autopsy/centralrepository/license-centralrepository.txt deleted file mode 100755 index ebd777cdad..0000000000 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/license-centralrepository.txt +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Central Repository - * - * Copyright 2015-2017 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/ImportHashDatabaseDialog.java b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/ImportHashDatabaseDialog.java index d324ad3830..e041f87a03 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/ImportHashDatabaseDialog.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/ImportHashDatabaseDialog.java @@ -50,10 +50,10 @@ import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.openide.windows.WindowManager; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifact; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttribute; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.ModuleSettings; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactInstance; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import org.sleuthkit.autopsy.centralrepository.datamodel.EamGlobalFileInstance; import org.sleuthkit.autopsy.centralrepository.datamodel.EamGlobalSet; @@ -535,7 +535,7 @@ final class ImportHashDatabaseDialog extends javax.swing.JDialog { // Future, make UI handle more than the "FILES" type. try { EamDb dbManager = EamDb.getInstance(); - EamArtifact.Type contentType = dbManager.getCorrelationTypeById(EamArtifact.FILES_TYPE_ID); // get "FILES" type + CorrelationAttribute.Type contentType = dbManager.getCorrelationTypeById(CorrelationAttribute.FILES_TYPE_ID); // get "FILES" type // run in the background and close dialog SwingUtilities.invokeLater(new ImportHashDatabaseWorker(selectedFilePath, knownStatus, globalSetID, contentType)::execute); dispose(); @@ -577,9 +577,9 @@ final class ImportHashDatabaseDialog extends javax.swing.JDialog { private final TskData.FileKnown knownStatus; private final int globalSetID; private final ProgressHandle progress; - private final EamArtifact.Type contentType; + private final CorrelationAttribute.Type contentType; - public ImportHashDatabaseWorker(String filename, TskData.FileKnown knownStatus, int globalSetID, EamArtifact.Type contentType) throws EamDbException, UnknownHostException { + public ImportHashDatabaseWorker(String filename, TskData.FileKnown knownStatus, int globalSetID, CorrelationAttribute.Type contentType) throws EamDbException, UnknownHostException { this.file = new File(filename); this.knownStatus = knownStatus; this.globalSetID = globalSetID; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/ManageCorrelationPropertiesDialog.java b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/ManageCorrelationPropertiesDialog.java index 0d45c51d2f..d0bfd114e7 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/ManageCorrelationPropertiesDialog.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/ManageCorrelationPropertiesDialog.java @@ -32,7 +32,7 @@ import org.openide.util.Exceptions; import org.openide.util.NbBundle.Messages; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifact; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttribute; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; @@ -44,7 +44,7 @@ final class ManageCorrelationPropertiesDialog extends javax.swing.JDialog { private static final Logger LOGGER = Logger.getLogger(ManageCorrelationPropertiesDialog.class.getName()); - private final List correlationTypes; + private final List correlationTypes; /** * Displays a dialog that allows a user to select which Type(s) should be @@ -73,7 +73,7 @@ final class ManageCorrelationPropertiesDialog extends javax.swing.JDialog { try { EamDb dbManager = EamDb.getInstance(); correlationTypes.clear(); - correlationTypes.addAll(dbManager.getCorrelationTypes()); + correlationTypes.addAll(dbManager.getDefinedCorrelationTypes()); } catch (EamDbException ex) { Exceptions.printStackTrace(ex); } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/ManageTagsDialog.java b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/ManageTagsDialog.java index 8f65bbf61d..1960ee3df4 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/ManageTagsDialog.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/ManageTagsDialog.java @@ -37,7 +37,7 @@ import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.services.TagsManager; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifact; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttribute; import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.TskCoreException; @@ -336,9 +336,8 @@ final class ManageTagsDialog extends javax.swing.JDialog { List artifactTags = curCase.getSleuthkitCase().getBlackboardArtifactTagsByTagName(tagName); for(BlackboardArtifactTag bbTag:artifactTags){ - List convertedArtifacts = EamArtifactUtil.fromBlackboardArtifact(bbTag.getArtifact(), true, - EamDb.getInstance().getCorrelationTypes(), true); - for (EamArtifact eamArtifact : convertedArtifacts) { + List convertedArtifacts = EamArtifactUtil.getCorrelationAttributeFromBlackboardArtifact(bbTag.getArtifact(), true, true); + for (CorrelationAttribute eamArtifact : convertedArtifacts) { EamDb.getInstance().setArtifactInstanceKnownStatus(eamArtifact,TskData.FileKnown.BAD); } } @@ -346,9 +345,11 @@ final class ManageTagsDialog extends javax.swing.JDialog { // Now search for files List fileTags = curCase.getSleuthkitCase().getContentTagsByTagName(tagName); for(ContentTag contentTag:fileTags){ - final EamArtifact eamArtifact = EamArtifactUtil.getEamArtifactFromContent(contentTag.getContent(), + final CorrelationAttribute eamArtifact = EamArtifactUtil.getEamArtifactFromContent(contentTag.getContent(), TskData.FileKnown.BAD, ""); - EamDb.getInstance().setArtifactInstanceKnownStatus(eamArtifact, TskData.FileKnown.BAD); + if(eamArtifact != null){ + EamDb.getInstance().setArtifactInstanceKnownStatus(eamArtifact, TskData.FileKnown.BAD); + } } } catch (TskCoreException ex){ throw new EamDbException("Error updating artifacts", ex); diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ThumbnailViewChildren.java b/Core/src/org/sleuthkit/autopsy/corecomponents/ThumbnailViewChildren.java index 50e15c8116..cd7e6e7f76 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ThumbnailViewChildren.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ThumbnailViewChildren.java @@ -55,6 +55,7 @@ import org.sleuthkit.autopsy.corecomponents.ResultViewerPersistence.SortCriterio import static org.sleuthkit.autopsy.corecomponents.ResultViewerPersistence.loadSortCriteria; import org.sleuthkit.autopsy.coreutils.ImageUtils; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; /** @@ -204,7 +205,7 @@ class ThumbnailViewChildren extends Children.Keys { private static boolean isSupported(Node node) { if (node != null) { - Content content = node.getLookup().lookup(Content.class); + Content content = node.getLookup().lookup(AbstractFile.class); if (content != null) { return ImageUtils.thumbnailSupported(content); } @@ -264,7 +265,7 @@ class ThumbnailViewChildren extends Children.Keys { private ThumbnailViewNode(Node wrappedNode, int thumbSize) { super(wrappedNode, FilterNode.Children.LEAF); this.thumbSize = thumbSize; - this.content = this.getLookup().lookup(Content.class); + this.content = this.getLookup().lookup(AbstractFile.class); } @Override diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/CreditCards.java b/Core/src/org/sleuthkit/autopsy/datamodel/CreditCards.java index 300eb118dc..d353df6c88 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/CreditCards.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/CreditCards.java @@ -1,3 +1,21 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2017 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.sleuthkit.autopsy.datamodel; import com.google.common.collect.Range; @@ -91,6 +109,8 @@ public class CreditCards { } private static final Logger LOGGER = Logger.getLogger(CreditCards.class.getName()); + + /** * Range Map from a (ranges of) BINs to data model object with details of * the BIN, ie, bank name, phone, url, visa/amex/mastercard/..., diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterChildren.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterChildren.java index 4ca9e458c6..959b2c6653 100755 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterChildren.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterChildren.java @@ -286,17 +286,17 @@ class DirectoryTreeFilterChildren extends FilterNode.Children { @Override public Boolean visit(LocalFileNode lfn) { - return lfn.hasContentChildren(); + return lfn.hasVisibleContentChildren(); } @Override public Boolean visit(LayoutFileNode ln) { - return ln.hasContentChildren(); + return ln.hasVisibleContentChildren(); } @Override public Boolean visit(SlackFileNode sfn) { - return sfn.hasContentChildren(); + return sfn.hasVisibleContentChildren(); } @Override diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobLogger.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobLogger.java index 122433c56b..dfff014556 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobLogger.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobLogger.java @@ -349,19 +349,6 @@ final class AutoIngestJobLogger { log(MessageCategory.WARNING, "Analysis of data source cancelled"); } - /** - * Logs that automated file export is not enabled. - * - * @throws AutoIngestJobLoggerException if there is an error writing the log - * message. - * @throws InterruptedException if interrupted while blocked waiting - * to acquire an exclusive lock on the - * log file. - */ - void logFileExportDisabled() throws AutoIngestJobLoggerException, InterruptedException { - log(MessageCategory.WARNING, "Automated file export is not enabled"); - } - /** * Logs completion of file export. * diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 5d59f01027..359f270af0 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -2533,10 +2533,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang if (fileExporter.isEnabled()) { fileExporter.process(manifest.getDeviceId(), dataSource.getContent(), currentJob::isCanceled); jobLogger.logFileExportCompleted(); - } else { - SYS_LOGGER.log(Level.WARNING, "Exporting files not enabled for {0}", manifestPath); - jobLogger.logFileExportDisabled(); - } + } } catch (FileExportException ex) { SYS_LOGGER.log(Level.SEVERE, String.format("Error doing file export for %s", manifestPath), ex); currentJob.setErrorsOccurred(true); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/CreditCardValidator.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/CreditCardValidator.java new file mode 100644 index 0000000000..baa8c17c44 --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/CreditCardValidator.java @@ -0,0 +1,159 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2017 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import com.google.common.base.CharMatcher; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.validator.routines.checkdigit.LuhnCheckDigit; + +/** + * Utility class to validate Credit Card Numbers. Validation entails checking + * that numbers are compatible but not necessarily 'real'. Validation checks the + * following properties: + *
    + *
  • A number can have obly one of dashes, spaces, or none as a seperator + * character.
    • + *
  • If a number has seperator character, the digits must be grouped into a + * valid pattern for the number length
    • + *
  • A number must pass the luhn check.
    • + *
+ * + */ +final class CreditCardValidator { + + private CreditCardValidator() { + } + + private static final LuhnCheckDigit CREDIT_CARD_NUM_LUHN_CHECK = new LuhnCheckDigit(); + + /** + * Does the given string represent a valid credit card number? It must have + * no separators, or only '-', or only ' '. Checks digit grouping for + * 15,16,and 19 digit numbers. All other length numbers must be contiguous + * or begin with a group of 4 digits. + * + * @param rawCCN + * + * @return True if rawCCN represents a valid credit card number. + */ + static public boolean isValidCCN(String rawCCN) { + //check for a valid separator + boolean hasSpace = StringUtils.contains(rawCCN, ' '); + boolean hasDash = StringUtils.contains(rawCCN, '-'); + if (hasSpace && hasDash) { + return false; //can only have dashes or spaces, not both. + } + + Character separator = null; + if (hasSpace) { + separator = ' '; + } else if (hasDash) { + separator = '-'; + } + + final String cannonicalCCN; + String[] splitCCN; + if (separator != null) { + //there is a seperator, strip if for canoncial form of CCN + cannonicalCCN = CharMatcher.anyOf(separator.toString()).removeFrom(rawCCN); + splitCCN = rawCCN.split(separator.toString()); + } else { + //else use 'defualt'values + cannonicalCCN = rawCCN; + splitCCN = new String[]{cannonicalCCN}; + } + + // validate digit grouping for 15, 16, and 19 digit cards + switch (cannonicalCCN.length()) { + case 15: + if (false == isValid15DigitGrouping(splitCCN)) { + return false; + } + break; + case 16: + if (false == isValid16DigitGrouping(splitCCN)) { + return false; + } + break; + case 19: + if (false == isValid19DigitGrouping(splitCCN)) { + return false; + } + break; + default: + if (false == isValidOtherDigitGrouping(splitCCN)) { + return false; + } + } + + return CREDIT_CARD_NUM_LUHN_CHECK.isValid(cannonicalCCN); + } + + static private boolean isValidOtherDigitGrouping(String[] splitCCN) { + if (splitCCN.length == 1) { + return true; + } else { + return splitCCN[0].length() == 4; + } + } + + static private boolean isValid19DigitGrouping(String[] splitCCN) { + switch (splitCCN.length) { + case 1: + return true; + case 2: + return splitCCN[0].length() == 6 + && splitCCN[1].length() == 13; + case 5: + return splitCCN[0].length() == 4 + && splitCCN[1].length() == 4 + && splitCCN[2].length() == 4 + && splitCCN[3].length() == 4 + && splitCCN[4].length() == 3; + default: + return false; + } + } + + static private boolean isValid16DigitGrouping(String[] splitCCN) { + switch (splitCCN.length) { + case 1: + return true; + case 4: + return splitCCN[0].length() == 4 + && splitCCN[1].length() == 4 + && splitCCN[2].length() == 4 + && splitCCN[3].length() == 4; + default: + return false; + } + } + + static private boolean isValid15DigitGrouping(String[] splitCCN) { + switch (splitCCN.length) { + case 1: + return true; + case 3: + return (splitCCN[0].length() == 4 && splitCCN[1].length() == 6 && splitCCN[2].length() == 5); +// UATP || ((splitCCN[0].length() == 4 && splitCCN[1].length() == 5 && splitCCN[2].length() == 6)); + default: + return false; + } + } +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchList.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchList.java index db8ceeb59c..8d7ef916b1 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchList.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchList.java @@ -51,13 +51,12 @@ abstract class KeywordSearchList { private static final String URL_REGEX = "(((((h|H)(t|T))|(f|F))(t|T)(p|P)(s|S?)\\:\\/\\/)|(w|W){3,3}\\.)[a-zA-Z0-9\\-\\.]+\\.([a-zA-Z]{2,5})(\\:[0-9]+)*(\\/($|[a-zA-Z0-9\\.\\,\\;\\?\\'\\\\+&%\\$#\\=~_\\-]+))*"; //NON-NLS /** - * 12-19 digits, with possible single spaces or dashes in between, optionally - * preceded by % (start sentinel) and a B (format code). - * Note that this regular expression is intentionally more broad than the - * regular expression used by the code that validates credit card account - * numbers. This regex used to attempt to limit hits to numbers starting - * with the digits 3 through 6 but this resulted in an error when we - * moved to Solr 6. + * 12-19 digits, with possible single spaces or dashes in between, + * optionally preceded by % (start sentinel) and a B (format code). Note + * that this regular expression is intentionally more broad than the regular + * expression used by the code that validates credit card account numbers. + * This regex used to attempt to limit hits to numbers starting with the + * digits 3 through 6 but this resulted in an error when we moved to Solr 6. */ private static final String CCN_REGEX = "(%?)(B?)([0-9][ \\-]*?){12,19}(\\^?)"; //NON-NLS diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java index a422d9f9c3..1642f13b8a 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java @@ -27,7 +27,7 @@ import java.util.Map; import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.validator.routines.DomainValidator; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery.SortClause; @@ -47,7 +47,9 @@ import static org.sleuthkit.autopsy.keywordsearch.TermsComponentQuery.KEYWORD_SE import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -69,31 +71,33 @@ import org.sleuthkit.datamodel.TskData; final class RegexQuery implements KeywordSearchQuery { public static final Logger LOGGER = Logger.getLogger(RegexQuery.class.getName()); - private final List filters = new ArrayList<>(); - private final KeywordList keywordList; - private final Keyword originalKeyword; // The regular expression originalKeyword used to perform the search. - private String field = Server.Schema.CONTENT_STR.toString(); - private final String keywordString; - static final private int MAX_RESULTS_PER_CURSOR_MARK = 512; - private boolean escaped; - private String escapedQuery; - - private final int MIN_EMAIL_ADDR_LENGTH = 8; - - // Lucene regular expressions do not support the following Java predefined - // and POSIX character classes. There are other valid Java character classes - // that are not supported by Lucene but we do not check for all of them. - // See https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html - // for Java regex syntax. - // See https://lucene.apache.org/core/6_4_0/core/org/apache/lucene/util/automaton/RegExp.html - // for Lucene syntax. - // We use \p as a shortcut for all of the character classes of the form \p{XXX}. + /** + * Lucene regular expressions do not support the following Java predefined + * and POSIX character classes. There are other valid Java character classes + * that are not supported by Lucene but we do not check for all of them. See + * https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html + * for Java regex syntax. See + * https://lucene.apache.org/core/6_4_0/core/org/apache/lucene/util/automaton/RegExp.html + * for Lucene syntax. We use \p as a shortcut for all of the character + * classes of the form \p{XXX}. + */ private static final CharSequence[] UNSUPPORTED_CHARS = {"\\d", "\\D", "\\w", "\\W", "\\s", "\\S", "\\n", "\\t", "\\r", "\\f", "\\a", "\\e", "\\v", "\\V", "\\h", "\\H", "\\p"}; //NON-NLS - private boolean queryStringContainsWildcardPrefix = false; - private boolean queryStringContainsWildcardSuffix = false; + private static final int MAX_RESULTS_PER_CURSOR_MARK = 512; + private static final int MIN_EMAIL_ADDR_LENGTH = 8; + + private final List filters = new ArrayList<>(); + private final KeywordList keywordList; + private final Keyword originalKeyword; // The regular expression originalKeyword used to perform the search. + private final String keywordString; + private final boolean queryStringContainsWildcardPrefix; + private final boolean queryStringContainsWildcardSuffix; + + private boolean escaped; + private String escapedQuery; + private String field = Server.Schema.CONTENT_STR.toString(); /** * Constructor with query to process. @@ -106,13 +110,8 @@ final class RegexQuery implements KeywordSearchQuery { this.originalKeyword = keyword; this.keywordString = keyword.getSearchTerm(); - if (this.keywordString.startsWith(".*")) { - this.queryStringContainsWildcardPrefix = true; - } - - if (this.keywordString.endsWith(".*")) { - this.queryStringContainsWildcardSuffix = true; - } + this.queryStringContainsWildcardPrefix = this.keywordString.startsWith(".*"); + this.queryStringContainsWildcardSuffix = this.keywordString.endsWith(".*"); } @Override @@ -252,6 +251,7 @@ final class RegexQuery implements KeywordSearchQuery { String hit = hitMatcher.group(); offset = hitMatcher.end(); + final ATTRIBUTE_TYPE artifactAttributeType = originalKeyword.getArtifactAttributeType(); // We attempt to reduce false positives for phone numbers and IP address hits // by querying Solr for hits delimited by a set of known boundary characters. @@ -260,9 +260,9 @@ final class RegexQuery implements KeywordSearchQuery { // needs to be chopped off, unless the user has supplied their own wildcard suffix // as part of the regex. if (!queryStringContainsWildcardSuffix - && (originalKeyword.getArtifactAttributeType() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER - || originalKeyword.getArtifactAttributeType() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IP_ADDRESS)) { - if (originalKeyword.getArtifactAttributeType() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER) { + && (artifactAttributeType == ATTRIBUTE_TYPE.TSK_PHONE_NUMBER + || artifactAttributeType == ATTRIBUTE_TYPE.TSK_IP_ADDRESS)) { + if (artifactAttributeType == ATTRIBUTE_TYPE.TSK_PHONE_NUMBER) { // For phone numbers replace all non numeric characters (except "(") at the start of the hit. hit = hit.replaceAll("^[^0-9\\(]", ""); } else { @@ -273,44 +273,48 @@ final class RegexQuery implements KeywordSearchQuery { hit = hit.replaceAll("[^0-9]$", ""); } - if (originalKeyword.getArtifactAttributeType() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL) { - // Reduce false positives by eliminating email address hits that are either - // too short or are not for valid top level domains. - if (hit.length() < MIN_EMAIL_ADDR_LENGTH - || !DomainValidator.getInstance(true).isValidTld(hit.substring(hit.lastIndexOf('.')))) { - continue; + if (artifactAttributeType == null) { + addHit(content, snippet, hitMatcher, hit, hits, docId); + } else { + switch (artifactAttributeType) { + case TSK_EMAIL: + /* + * Reduce false positives by eliminating email + * address hits that are either too short or are + * not for valid top level domains. + */ + if (hit.length() >= MIN_EMAIL_ADDR_LENGTH + && DomainValidator.getInstance(true).isValidTld(hit.substring(hit.lastIndexOf('.')))) { + addHit(content, snippet, hitMatcher, hit, hits, docId); + } + + break; + case TSK_CARD_NUMBER: + /* + * If searching for credit card account numbers, + * do extra validation on the term and discard + * it if it does not pass. + */ + Matcher ccnMatcher = CREDIT_CARD_NUM_PATTERN.matcher(hit); + + for (int rLength = hit.length(); rLength >= 12; rLength--) { + ccnMatcher.region(0, rLength); + if (ccnMatcher.find()) { + final String group = ccnMatcher.group("ccn"); + if (CreditCardValidator.isValidCCN(group)) { + addHit(content, snippet, hitMatcher, hit, hits, docId); + }; + } + } + + break; + default: + addHit(content, snippet, hitMatcher, hit, hits, docId); + } } - - /* - * If searching for credit card account numbers, do a Luhn - * check on the term and discard it if it does not pass. - */ - if (originalKeyword.getArtifactAttributeType() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CARD_NUMBER) { - Matcher ccnMatcher = CREDIT_CARD_NUM_PATTERN.matcher(hit); - if (ccnMatcher.find()) { - final String ccn = CharMatcher.anyOf(" -").removeFrom(ccnMatcher.group("ccn")); - if (false == TermsComponentQuery.CREDIT_CARD_NUM_LUHN_CHECK.isValid(ccn)) { - continue; - } - } else { - continue; - } - } - - /** - * Get the snippet from the document if keyword search is - * configured to use snippets. - */ - int maxIndex = content.length() - 1; - snippet.append(content.substring(Integer.max(0, hitMatcher.start() - 20), Integer.max(0, hitMatcher.start()))); - snippet.appendCodePoint(171); - snippet.append(hit); - snippet.appendCodePoint(171); - snippet.append(content.substring(Integer.min(maxIndex, hitMatcher.end()), Integer.min(maxIndex, hitMatcher.end() + 20))); - - hits.add(new KeywordHit(docId, snippet.toString(), hit)); } + } } catch (TskCoreException ex) { throw ex; @@ -328,6 +332,21 @@ final class RegexQuery implements KeywordSearchQuery { return hits; } + private void addHit(String content, StringBuilder snippet, Matcher hitMatcher, String hit, List hits, final String docId) throws TskCoreException { + /** + * Get the snippet from the document if keyword search is configured to + * use snippets. + */ + int maxIndex = content.length() - 1; + snippet.append(content.substring(Integer.max(0, hitMatcher.start() - 20), Integer.max(0, hitMatcher.start()))); + snippet.appendCodePoint(171); + snippet.append(hit); + snippet.appendCodePoint(171); + snippet.append(content.substring(Integer.min(maxIndex, hitMatcher.end()), Integer.min(maxIndex, hitMatcher.end() + 20))); + + hits.add(new KeywordHit(docId, snippet.toString(), hit)); + } + @Override public void addFilter(KeywordQueryFilter filter) { this.filters.add(filter); @@ -390,11 +409,11 @@ final class RegexQuery implements KeywordSearchQuery { */ BlackboardArtifact newArtifact; Collection attributes = new ArrayList<>(); - if (originalKeyword.getArtifactAttributeType() != BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CARD_NUMBER) { - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, foundKeyword.getSearchTerm())); - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, getQueryString())); + if (originalKeyword.getArtifactAttributeType() != ATTRIBUTE_TYPE.TSK_CARD_NUMBER) { + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, foundKeyword.getSearchTerm())); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, getQueryString())); try { - newArtifact = content.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT); + newArtifact = content.newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT); } catch (TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error adding artifact for keyword hit to blackboard", ex); //NON-NLS return null; @@ -404,7 +423,7 @@ final class RegexQuery implements KeywordSearchQuery { * Parse the credit card account attributes from the snippet for the * hit. */ - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE, MODULE_NAME, Account.Type.CREDIT_CARD.name())); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE, MODULE_NAME, Account.Type.CREDIT_CARD.name())); Map parsedTrackAttributeMap = new HashMap<>(); Matcher matcher = TermsComponentQuery.CREDIT_CARD_TRACK1_PATTERN.matcher(hit.getSnippet()); if (matcher.find()) { @@ -414,7 +433,7 @@ final class RegexQuery implements KeywordSearchQuery { if (matcher.find()) { parseTrack2Data(parsedTrackAttributeMap, matcher); } - final BlackboardAttribute ccnAttribute = parsedTrackAttributeMap.get(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CARD_NUMBER)); + final BlackboardAttribute ccnAttribute = parsedTrackAttributeMap.get(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_CARD_NUMBER)); if (ccnAttribute == null || StringUtils.isBlank(ccnAttribute.getValueString())) { if (hit.isArtifactHit()) { LOGGER.log(Level.SEVERE, String.format("Failed to parse credit card account number for artifact keyword hit: term = %s, snippet = '%s', artifact id = %d", foundKeyword.getSearchTerm(), hit.getSnippet(), hit.getArtifactID().get())); //NON-NLS @@ -433,21 +452,21 @@ final class RegexQuery implements KeywordSearchQuery { CreditCards.BankIdentificationNumber binInfo = CreditCards.getBINInfo(bin); if (binInfo != null) { binInfo.getScheme().ifPresent(scheme - -> attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CARD_SCHEME, MODULE_NAME, scheme))); + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CARD_SCHEME, MODULE_NAME, scheme))); binInfo.getCardType().ifPresent(cardType - -> attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CARD_TYPE, MODULE_NAME, cardType))); + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CARD_TYPE, MODULE_NAME, cardType))); binInfo.getBrand().ifPresent(brand - -> attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_BRAND_NAME, MODULE_NAME, brand))); + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_BRAND_NAME, MODULE_NAME, brand))); binInfo.getBankName().ifPresent(bankName - -> attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_BANK_NAME, MODULE_NAME, bankName))); + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_BANK_NAME, MODULE_NAME, bankName))); binInfo.getBankPhoneNumber().ifPresent(phoneNumber - -> attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, MODULE_NAME, phoneNumber))); + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, MODULE_NAME, phoneNumber))); binInfo.getBankURL().ifPresent(url - -> attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL, MODULE_NAME, url))); + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL, MODULE_NAME, url))); binInfo.getCountry().ifPresent(country - -> attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COUNTRY, MODULE_NAME, country))); + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNTRY, MODULE_NAME, country))); binInfo.getBankCity().ifPresent(city - -> attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CITY, MODULE_NAME, city))); + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CITY, MODULE_NAME, city))); } /* @@ -467,7 +486,7 @@ final class RegexQuery implements KeywordSearchQuery { * Create an account artifact. */ try { - newArtifact = content.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ACCOUNT); + newArtifact = content.newArtifact(ARTIFACT_TYPE.TSK_ACCOUNT); } catch (TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error adding artifact for account to blackboard", ex); //NON-NLS return null; @@ -475,17 +494,17 @@ final class RegexQuery implements KeywordSearchQuery { } if (StringUtils.isNotBlank(listName)) { - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName)); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName)); } if (snippet != null) { - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet)); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet)); } - + hit.getArtifactID().ifPresent(artifactID - -> attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID)) + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID)) ); - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.REGEX.ordinal())); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.REGEX.ordinal())); try { newArtifact.addAttributes(attributes); @@ -505,11 +524,11 @@ final class RegexQuery implements KeywordSearchQuery { * @param matcher A matcher for the snippet. */ static private void parseTrack2Data(Map attributesMap, Matcher matcher) { - addAttributeIfNotAlreadyCaptured(attributesMap, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CARD_NUMBER, "accountNumber", matcher); - addAttributeIfNotAlreadyCaptured(attributesMap, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CARD_EXPIRATION, "expiration", matcher); - addAttributeIfNotAlreadyCaptured(attributesMap, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CARD_SERVICE_CODE, "serviceCode", matcher); - addAttributeIfNotAlreadyCaptured(attributesMap, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CARD_DISCRETIONARY, "discretionary", matcher); - addAttributeIfNotAlreadyCaptured(attributesMap, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CARD_LRC, "LRC", matcher); + addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_NUMBER, "accountNumber", matcher); + addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_EXPIRATION, "expiration", matcher); + addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_SERVICE_CODE, "serviceCode", matcher); + addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_DISCRETIONARY, "discretionary", matcher); + addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_LRC, "LRC", matcher); } /** @@ -518,12 +537,12 @@ final class RegexQuery implements KeywordSearchQuery { * same fields as the track two data, plus the account holder's name. * * @param attributeMap A map of artifact attribute objects, used to avoid - * creating duplicate attributes. - * @param matcher A matcher for the snippet. + * creating duplicate attributes. + * @param matcher A matcher for the snippet. */ static private void parseTrack1Data(Map attributeMap, Matcher matcher) { parseTrack2Data(attributeMap, matcher); - addAttributeIfNotAlreadyCaptured(attributeMap, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME_PERSON, "name", matcher); + addAttributeIfNotAlreadyCaptured(attributeMap, ATTRIBUTE_TYPE.TSK_NAME_PERSON, "name", matcher); } /** @@ -531,20 +550,20 @@ final class RegexQuery implements KeywordSearchQuery { * value parsed from the snippet for a credit account number hit. * * @param attributeMap A map of artifact attribute objects, used to avoid - * creating duplicate attributes. - * @param attrType The type of attribute to create. - * @param groupName The group name of the regular expression that was - * used to parse the attribute data. - * @param matcher A matcher for the snippet. - + * creating duplicate attributes. + * @param attrType The type of attribute to create. + * @param groupName The group name of the regular expression that was + * used to parse the attribute data. + * @param matcher A matcher for the snippet. + * */ - static private void addAttributeIfNotAlreadyCaptured(Map attributeMap, BlackboardAttribute.ATTRIBUTE_TYPE attrType, String groupName, Matcher matcher) { + static private void addAttributeIfNotAlreadyCaptured(Map attributeMap, ATTRIBUTE_TYPE attrType, String groupName, Matcher matcher) { BlackboardAttribute.Type type = new BlackboardAttribute.Type(attrType); attributeMap.computeIfAbsent(type, (BlackboardAttribute.Type t) -> { String value = matcher.group(groupName); - if (attrType.equals(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CARD_NUMBER)) { - attributeMap.put(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD), - new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, value)); + if (attrType.equals(ATTRIBUTE_TYPE.TSK_CARD_NUMBER)) { + attributeMap.put(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_KEYWORD), + new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, value)); value = CharMatcher.anyOf(" -").removeFrom(value); } if (StringUtils.isNotBlank(value)) { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsComponentQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsComponentQuery.java index f37be9952e..58087c4590 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsComponentQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsComponentQuery.java @@ -30,7 +30,6 @@ import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang.StringUtils; -import org.apache.commons.validator.routines.checkdigit.LuhnCheckDigit; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.response.TermsResponse.Term; import org.sleuthkit.autopsy.coreutils.Logger; @@ -62,19 +61,26 @@ final class TermsComponentQuery implements KeywordSearchQuery { private static final String CASE_INSENSITIVE = "case_insensitive"; //NON-NLS private static final boolean DEBUG_FLAG = Version.Type.DEVELOPMENT.equals(Version.getBuildType()); private static final int MAX_TERMS_QUERY_RESULTS = 20000; + private final KeywordList keywordList; private final Keyword originalKeyword; + private final List filters = new ArrayList<>(); // THIS APPEARS TO BE UNUSED + private String searchTerm; private boolean searchTermIsEscaped; - private final List filters = new ArrayList<>(); // THIS APPEARS TO BE UNUSED /* * The following fields are part of the initial implementation of credit * card account search and should be factored into another class when time * permits. */ - static final Pattern CREDIT_CARD_NUM_PATTERN = Pattern.compile("(?[3-6]([ -]?[0-9]){11,18})"); //12-19 digits, with possible single spaces or dashes in between. First digit is 3,4,5, or 6 //NON-NLS - static final LuhnCheckDigit CREDIT_CARD_NUM_LUHN_CHECK = new LuhnCheckDigit(); + /** + * 12-19 digits, with possible single spaces or dashes in between. First + * digit is 2 through 6 + * + */ + static final Pattern CREDIT_CARD_NUM_PATTERN = + Pattern.compile("(?[2-6]([ -]?[0-9]){11,18})"); static final Pattern CREDIT_CARD_TRACK1_PATTERN = Pattern.compile( /* * Track 1 is alphanumeric. @@ -87,7 +93,7 @@ final class TermsComponentQuery implements KeywordSearchQuery { "(?:" //begin nested optinal group //NON-NLS + "%?" //optional start sentinal: % //NON-NLS + "B)?" //format code //NON-NLS - + "(?[3-6]([ -]?[0-9]){11,18})" //12-19 digits, with possible single spaces or dashes in between. first digit is 3,4,5, or 6 //NON-NLS + + "(?[2-6]([ -]?[0-9]){11,18})" //12-19 digits, with possible single spaces or dashes in between. first digit is 2,3,4,5, or 6 //NON-NLS + "\\^" //separator //NON-NLS + "(?[^^]{2,26})" //2-26 charachter name, not containing ^ //NON-NLS + "(?:\\^" //separator //NON-NLS @@ -108,7 +114,7 @@ final class TermsComponentQuery implements KeywordSearchQuery { * */ "[:;<=>?]?" //(optional)start sentinel //NON-NLS - + "(?[3-6]([ -]?[0-9]){11,18})" //12-19 digits, with possible single spaces or dashes in between. first digit is 3,4,5, or 6 //NON-NLS + + "(?[2-6]([ -]?[0-9]){11,18})" //12-19 digits, with possible single spaces or dashes in between. first digit is 2,3,4,5, or 6 //NON-NLS + "(?:[:;<=>?]" //separator //NON-NLS + "(?:(?\\d{4})" //4 digit expiration date YYMM //NON-NLS + "(?:(?\\d{3})" //3 digit service code //NON-NLS @@ -118,6 +124,7 @@ final class TermsComponentQuery implements KeywordSearchQuery { + "?)?)?)?)?)?"); //close nested optional groups //NON-NLS static final BlackboardAttribute.Type KEYWORD_SEARCH_DOCUMENT_ID = new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_DOCUMENT_ID); + /** * Constructs an object that implements a regex query that will be performed * as a two step operation. In the first step, the Solr terms component is @@ -289,9 +296,8 @@ final class TermsComponentQuery implements KeywordSearchQuery { */ if (originalKeyword.getArtifactAttributeType() == ATTRIBUTE_TYPE.TSK_CARD_NUMBER) { Matcher matcher = CREDIT_CARD_NUM_PATTERN.matcher(term.getTerm()); - matcher.find(); - final String ccn = CharMatcher.anyOf(" -").removeFrom(matcher.group("ccn")); - if (false == CREDIT_CARD_NUM_LUHN_CHECK.isValid(ccn)) { + if (false == matcher.find() + || false == CreditCardValidator.isValidCCN(matcher.group("ccn"))) { continue; } } @@ -319,7 +325,6 @@ final class TermsComponentQuery implements KeywordSearchQuery { return results; } - @Override public BlackboardArtifact writeSingleFileHitsToBlackBoard(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) { /* @@ -460,9 +465,9 @@ final class TermsComponentQuery implements KeywordSearchQuery { * hit and turns them into artifact attributes. The track 1 data has the * same fields as the track two data, plus the account holder's name. * - * @param attributesMap A map of artifact attribute objects, used to avoid - * creating duplicate attributes. - * @param matcher A matcher for the snippet. + * @param attributeMap A map of artifact attribute objects, used to avoid + * creating duplicate attributes. + * @param matcher A matcher for the snippet. */ static private void parseTrack1Data(Map attributeMap, Matcher matcher) { parseTrack2Data(attributeMap, matcher); diff --git a/KeywordSearch/test/unit/src/org/sleuthkit/autopsy/keywordsearch/CreditCardValidatorTest.java b/KeywordSearch/test/unit/src/org/sleuthkit/autopsy/keywordsearch/CreditCardValidatorTest.java new file mode 100644 index 0000000000..637be8b3ae --- /dev/null +++ b/KeywordSearch/test/unit/src/org/sleuthkit/autopsy/keywordsearch/CreditCardValidatorTest.java @@ -0,0 +1,190 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2017 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import org.junit.After; +import org.junit.AfterClass; +import static org.junit.Assert.assertEquals; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class CreditCardValidatorTest { + + public CreditCardValidatorTest() { + } + + @BeforeClass + public static void setUpClass() { + } + + @AfterClass + public static void tearDownClass() { + } + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + /** + * Test of isValidCCN method, of class CreditCardValidator. + */ + @Test + public void testIsValidCCN16() { + System.out.println("isValidCCN"); + + //rules for separators and grouping for 16 digits + assertEquals(true, CreditCardValidator.isValidCCN("1234567890318342"));// dashes + assertEquals(true, CreditCardValidator.isValidCCN("1234-5678-9031-8342"));// dashes + assertEquals(true, CreditCardValidator.isValidCCN("1234 5678 9031 8342"));// or spaces + + assertEquals(false, CreditCardValidator.isValidCCN("1234567890318341"));// luhn + assertEquals(false, CreditCardValidator.isValidCCN("1234-5678-9031 8342")); //only one seperator + assertEquals(false, CreditCardValidator.isValidCCN("1234-5678-90-318342")); //only four groups of four + assertEquals(false, CreditCardValidator.isValidCCN("1234 5678 90 318342")); //only four groups of four + assertEquals(false, CreditCardValidator.isValidCCN("1-2-3-4-5-6-7-8-9-0-3-1-8-3-4-2")); //only four groups of four + } + + @Test + public void testIsValidCCN15() { + System.out.println("isValidCCN"); + + //amex are fifteen digits, and grouped 4 6 5 + //amex cards that strart with 34 + assertEquals(true, CreditCardValidator.isValidCCN("3431 136294 58529")); + assertEquals(true, CreditCardValidator.isValidCCN("3431-136294-58529")); + assertEquals(true, CreditCardValidator.isValidCCN("343113629458529")); + + assertEquals(false, CreditCardValidator.isValidCCN("343113629458528")); //luhn + assertEquals(false, CreditCardValidator.isValidCCN("3431 13629458 529")); //grouping + assertEquals(false, CreditCardValidator.isValidCCN("3431 136294-58529")); //separators + + //amex cards that start with 37 + assertEquals(true, CreditCardValidator.isValidCCN("377585291285489")); + assertEquals(true, CreditCardValidator.isValidCCN("3775-852912-85489")); + assertEquals(true, CreditCardValidator.isValidCCN("3775 852912 85489")); + + assertEquals(false, CreditCardValidator.isValidCCN("377585291285488")); //luhn + assertEquals(false, CreditCardValidator.isValidCCN("3775-852912 85489")); //separator + assertEquals(false, CreditCardValidator.isValidCCN("37-7585-29-1285489")); //grouping + assertEquals(false, CreditCardValidator.isValidCCN("377585 29128548 9")); //grouping + + //UATP are also 15 digits, start with 1 and are typically 4-5-6 +// assertEquals(true, CreditCardValidator.isValidCCN("1409 56201 545229")); +// assertEquals(true, CreditCardValidator.isValidCCN("1409-56201-545229")); +// assertEquals(true, CreditCardValidator.isValidCCN("140956201545229")); +// assertEquals(false, CreditCardValidator.isValidCCN("140 9562015 45229")); +// assertEquals(false, CreditCardValidator.isValidCCN("1409-56201 545229")); + } + + @Test + public void testIsValidCCN19() { + System.out.println("isValidCCN"); + //nineteen digit (visa) cards 4-4-4-4-3 + assertEquals(true, CreditCardValidator.isValidCCN("4539747947839518654")); + assertEquals(true, CreditCardValidator.isValidCCN("4539-7479-4783-9518-654")); + assertEquals(true, CreditCardValidator.isValidCCN("4539 7479 4783 9518 654")); + + assertEquals(false, CreditCardValidator.isValidCCN("4539747947839518653")); //luhn + assertEquals(false, CreditCardValidator.isValidCCN("4539-7479 4783 9518 654")); //separators + assertEquals(false, CreditCardValidator.isValidCCN("45374 79 4783 9518 654")); //grouping + + //nineteen digit China UnionPay is 19 digits 6-13 (or 4-4-4-4) beging 62 + assertEquals(true, CreditCardValidator.isValidCCN("6239747947839518659")); + assertEquals(true, CreditCardValidator.isValidCCN("623974 7947839518659")); + assertEquals(true, CreditCardValidator.isValidCCN("623974-7947839518659")); + /* + * China UnionPay may not use luhn ??? + * + * https://stackoverflow.com/questions/7863058/does-the-luhn-algorithm-work-for-all-mainstream-credit-cards-discover-visa-m + */ + assertEquals(false, CreditCardValidator.isValidCCN("6239747947839518658")); //luhn + assertEquals(false, CreditCardValidator.isValidCCN("623974-79478395 18659")); //separators + assertEquals(false, CreditCardValidator.isValidCCN("62397-47947839518659")); //grouping + } + + @Test + public void testIsValidCCN18() { + System.out.println("isValidCCN"); + + assertEquals(true, CreditCardValidator.isValidCCN("123456789031834267")); + assertEquals(true, CreditCardValidator.isValidCCN("1234 5678 9031 8342 67")); + assertEquals(true, CreditCardValidator.isValidCCN("1234-56789031834-267")); + + assertEquals(false, CreditCardValidator.isValidCCN("123456789031834266")); //luhn + assertEquals(false, CreditCardValidator.isValidCCN("123 456789031834267")); //grouping + assertEquals(false, CreditCardValidator.isValidCCN("1234-56789 031834267")); //separators + } + + @Test + public void testIsValidCCN17() { + System.out.println("isValidCCN"); + + assertEquals(true, CreditCardValidator.isValidCCN("12345678903183426")); + assertEquals(true, CreditCardValidator.isValidCCN("1234 5678 9031 8342 6")); + assertEquals(true, CreditCardValidator.isValidCCN("1234-56789031834-26")); + + assertEquals(false, CreditCardValidator.isValidCCN("12345678903183425"));//luhn + assertEquals(false, CreditCardValidator.isValidCCN("123 45678903183426")); //grouping + assertEquals(false, CreditCardValidator.isValidCCN("1234-56789 03183426")); //separators + } + + @Test + public void testIsValidCCN14() { + System.out.println("isValidCCN"); + + assertEquals(true, CreditCardValidator.isValidCCN("12345678903183")); + assertEquals(true, CreditCardValidator.isValidCCN("1234 5678 9031 83")); + assertEquals(true, CreditCardValidator.isValidCCN("1234-5678903183")); + + assertEquals(false, CreditCardValidator.isValidCCN("12345678903182"));//luhn + assertEquals(false, CreditCardValidator.isValidCCN("123 45678903183")); //grouping + assertEquals(false, CreditCardValidator.isValidCCN("1234-56789 03183")); //separators + } + + @Test + public void testIsValidCCN13() { + System.out.println("isValidCCN"); + + assertEquals(true, CreditCardValidator.isValidCCN("1234567890318")); + assertEquals(true, CreditCardValidator.isValidCCN("1234 5678 9031 8")); + assertEquals(true, CreditCardValidator.isValidCCN("1234-567890318")); + + assertEquals(false, CreditCardValidator.isValidCCN("1234567890317"));//luhn + assertEquals(false, CreditCardValidator.isValidCCN("123 4567890318")); //grouping + assertEquals(false, CreditCardValidator.isValidCCN("1234-56789 0318")); //separators + } + + @Test + public void testIsValidCCN12() { + System.out.println("isValidCCN"); + + assertEquals(true, CreditCardValidator.isValidCCN("123456789031")); + assertEquals(true, CreditCardValidator.isValidCCN("1234 5678 9031")); + assertEquals(true, CreditCardValidator.isValidCCN("1234-56789031")); + + assertEquals(false, CreditCardValidator.isValidCCN("123456789030")); //luhn + assertEquals(false, CreditCardValidator.isValidCCN("123 456789031")); //grouping + assertEquals(false, CreditCardValidator.isValidCCN("1234-56789 031")); //separators + } +} diff --git a/build.xml b/build.xml index deda12f21e..bba4207f1b 100755 --- a/build.xml +++ b/build.xml @@ -263,9 +263,11 @@ - + + +