updates

Core/src/org/sleuthkit/autopsy/mainui/datamodel/EmailsDAO.java

@@ -65,19 +65,19 @@ import org.sleuthkit.datamodel.TskCoreException;
 import org.sleuthkit.datamodel.TskData;
 
 /**
- * Provides information to populate the results viewer for data in the
+ * Provides information to populate the results viewer for data in the Emails
- * Emails section.
+ * section.
  */
 public class EmailsDAO extends AbstractDAO {
 
     private static final Logger logger = Logger.getLogger(EmailsDAO.class.getName());
 
-    private static final String PATH_DELIMITER = "/";
+    private static final String PATH_DELIMITER = "\\";
-    private static final String ESCAPE_CHAR = "\\";
+    private static final String ESCAPE_CHAR = "$";
+
+    private final Cache<SearchParams<EmailSearchParams>, SearchResultsDTO> searchParamsCache
+            = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE).expireAfterAccess(CACHE_DURATION, CACHE_DURATION_UNITS).build();
 
-    private final Cache<SearchParams<EmailSearchParams>, SearchResultsDTO> searchParamsCache = 
-            CacheBuilder.newBuilder().maximumSize(CACHE_SIZE).expireAfterAccess(CACHE_DURATION, CACHE_DURATION_UNITS).build();
-    
     private final TreeCounts<EmailEvent> emailCounts = new TreeCounts<>();
 
     private static EmailsDAO instance = null;
@@ -195,14 +195,14 @@ public class EmailsDAO extends AbstractDAO {
 
             });
         }
-        
+
         Stream<Long> pagedIdStream = allMatchingIds.stream()
                 .skip(searchParams.getStartItem());
-        
+
         if (searchParams.getMaxResultsCount() != null && searchParams.getMaxResultsCount() > 0) {
             pagedIdStream = pagedIdStream.limit(searchParams.getMaxResultsCount());
         }
-        
+
         List<Long> pagedIds = pagedIdStream.collect(Collectors.toList());
 
         List<BlackboardArtifact> allArtifacts = Collections.emptyList();
@@ -274,83 +274,53 @@ public class EmailsDAO extends AbstractDAO {
      * @return A tuple of the sql and the account like string (or null if no
      *         account filter).
      */
-    private static Pair<String, String> getAccountFolderSql(TskData.DbType dbType, String account, Long dataSourceId) {
+    private static String getAccountFolderSql(TskData.DbType dbType, String account, Long dataSourceId) {
         // possible and claused depending on whether or not there is an account to filter on and a data source object id to filter on.
 
-        String accountClause = "";
+        String folderSplitSql;
-        if (account != null) {
-            if (StringUtils.isBlank(account)) {
-                accountClause = "      AND (attr.value_text IS NULL OR attr.value_text NOT LIKE '%/%/%/%' OR attr.value_text LIKE ?)\n";
-            } else {
-                accountClause = "      AND attr.value_text LIKE ? ESCAPE '" + ESCAPE_CHAR + "'\n";
-            }
-        }
-
-        String dataSourceClause = (dataSourceId == null ? "" : "      AND art.data_source_obj_id = ?\n");
-
-        // get path attribute value for emails 
-        String innerQuery = "SELECT\n"
-                + "      MIN(attr.value_text) AS path, \n"
-                + "      art.artifact_id\n"
-                + "    FROM blackboard_artifacts art\n"
-                + "    LEFT JOIN blackboard_attributes attr ON attr.artifact_id = art.artifact_id\n"
-                + "      AND attr.attribute_type_id = " + BlackboardAttribute.Type.TSK_PATH.getTypeID() + "\n" // may change due to JIRA-8220
-                + "    WHERE\n"
-                + " art.artifact_type_id = " + BlackboardArtifact.Type.TSK_EMAIL_MSG.getTypeID() + "\n"
-                + accountClause
-                + dataSourceClause
-                + "    GROUP BY art.artifact_id\n";
-
-        // get index 2 (account) and index 3 (folder) after splitting on delimiter
-        String accountFolderQuery;
         switch (dbType) {
             case POSTGRESQL:
-                accountFolderQuery = "SELECT\n"
+                folderSplitSql = "SPLITPART(res.subfolders, '\\', 1)";
-                        + (account != null ? "" : "  SPLIT_PART(a.path, '" + PATH_DELIMITER + "', 3) AS account,\n")
-                        + "  SPLIT_PART(a.path, '" + PATH_DELIMITER + "', 4) AS folder\n"
-                        + "FROM (\n"
-                        + innerQuery
-                        + "\n) a";
                 break;
             case SQLITE:
-                accountFolderQuery = "SELECT\n"
+                folderSplitSql = "SUBSTR(res.subfolders, 1, INSTR(res.subfolders, '\\') - 1)";
-                        + (account != null ? "" : "  a.account AS account,\n")
-                        + "  (CASE \n"
-                        + "    WHEN INSTR(a.remaining, '" + PATH_DELIMITER + "') > 0 THEN SUBSTR(a.remaining, 1, INSTR(a.remaining, '" + PATH_DELIMITER + "') - 1) \n"
-                        + "    ELSE a.remaining\n"
-                        + "  END) AS folder\n"
-                        + "FROM (\n"
-                        + "  SELECT \n"
-                        + "    SUBSTR(l.ltrimmed, 1, INSTR(l.ltrimmed, '" + PATH_DELIMITER + "') - 1) AS account,\n"
-                        + "    SUBSTR(l.ltrimmed, INSTR(l.ltrimmed, '" + PATH_DELIMITER + "') + 1) AS remaining\n"
-                        + "  FROM (\n"
-                        + "      SELECT SUBSTR(email_paths.path, INSTR(SUBSTR(email_paths.path, 2), '" + PATH_DELIMITER + "') + 2) AS ltrimmed\n"
-                        + "      FROM (\n"
-                        + innerQuery
-                        + "      ) email_paths\n"
-                        + "  ) l\n"
-                        + ") a\n";
                 break;
             default:
                 throw new IllegalArgumentException("Unknown db type: " + dbType);
         }
 
-        // group and get counts
+        String query = "\n  grouped_res.folder\n"
-        String sql = "  COUNT(*) AS count,\n "
+                + "  ,MAX(grouped_res.has_children) AS has_children\n"
-                + (account != null ? "" : "  account_folder.account,\n")
-                + "  account_folder.folder\n"
                 + "FROM (\n"
-                + accountFolderQuery
+                + "  SELECT\n"
-                + "\n) AS account_folder\n"
+                + "    (CASE \n"
-                + "GROUP BY \n"
+                + "        WHEN res.subfolders LIKE '%" + PATH_DELIMITER + "%' THEN \n"
-                + (account != null ? "" : "  account_folder.account,\n")
+                + "          " + folderSplitSql + "\n"
-                + "  account_folder.folder";
+                + "        ELSE\n"
+                + "          res.subfolders\n"
+                + "    END) AS folder\n"
+                + "    ,(CASE WHEN res.subfolders LIKE '%" + PATH_DELIMITER + "%' THEN 1 ELSE 0 END) AS has_children\n"
+                + "  FROM (\n"
+                + "    SELECT\n"
+                + "      SUBSTR(p.path, LENGTH(?) + 1) AS subfolders\n"
+                + "    FROM blackboard_artifacts art\n"
+                + "    LEFT JOIN (\n"
+                + "        SELECT \n"
+                + "        MIN(attr.value_text) AS path\n"
+                + "        ,attr.artifact_id \n"
+                + "        FROM blackboard_attributes attr \n"
+                + "        WHERE attr.attribute_type_id = " + BlackboardAttribute.Type.TSK_PATH.getTypeID() + " \n"
+                + "        GROUP BY attr.artifact_id\n"
+                + "    ) p ON art.artifact_id = p.artifact_id\n"
+                + "    WHERE p.path LIKE ? ESCAPE '" + ESCAPE_CHAR + "'\n"
+                + "    AND art.artifact_type_id = " + BlackboardArtifact.Type.TSK_EMAIL_MSG.getTypeID() + " \n"
+                + (dataSourceId != null ? "    AND art.data_source_obj_id = ? \n" : "")
+                + "  ) res\n"
+                + ") grouped_res\n"
+                + "GROUP BY grouped_res.folder\n"
+                + "ORDER BY grouped_res.folder";
 
-        String accountLikeStr = (account == null)
+        return query;
-                ? null
-                : "%/%/" + SubDAOUtils.likeEscape(account, ESCAPE_CHAR) + "/%";
-
-        return Pair.of(sql, accountLikeStr);
     }
 
     /**
@@ -363,7 +333,7 @@ public class EmailsDAO extends AbstractDAO {
      *
      * @throws ExecutionException
      */
-    public TreeResultsDTO<EmailSearchParams> getEmailCounts(Long dataSourceId, String account) throws ExecutionException {
+    public TreeResultsDTO<EmailSearchParams> getEmailCounts(Long dataSourceId, String folder) throws ExecutionException {
 
         // track indeterminate types by key (account if account is null, account folders if account parameter is non-null)
         Set<String> indeterminateTypes = this.emailCounts.getEnqueued().stream()
@@ -375,12 +345,11 @@ public class EmailsDAO extends AbstractDAO {
         String query = null;
         try {
             SleuthkitCase skCase = getCase();
-            Pair<String, String> sqlAndLike = getAccountFolderSql(skCase.getDatabaseType(), account, dataSourceId);
+            query = getAccountFolderSql(skCase.getDatabaseType(), folder, dataSourceId);
-            query = sqlAndLike.getLeft();
+
-            String accountLikeStr = sqlAndLike.getRight();
-            
             try (CaseDbPreparedStatement preparedStatement = skCase.getCaseDbAccessManager().prepareSelect(query)) {
 
+                
                 int paramIdx = 0;
                 if (account != null) {
                     preparedStatement.setString(++paramIdx, accountLikeStr);
@@ -406,10 +375,10 @@ public class EmailsDAO extends AbstractDAO {
                             return createEmailTreeItem(entry.getAccount(), entry.getFolder(), entry.getDisplayName(), dataSourceId, treeDisplayCount);
 
                         })
-                        .sorted((a,b) -> {
+                        .sorted((a, b) -> {
                             boolean keyADown = StringUtils.isBlank((account == null ? a.getSearchParams().getAccount() : a.getSearchParams().getFolder()));
                             boolean keyBDown = StringUtils.isBlank((account == null ? b.getSearchParams().getAccount() : b.getSearchParams().getFolder()));
-                                   
+
                             if (keyADown != keyBDown) {
                                 return Boolean.compare(keyADown, keyBDown);
                             } else {
@@ -478,10 +447,10 @@ public class EmailsDAO extends AbstractDAO {
                     if (StringUtils.isBlank(resultFolder) || StringUtils.isBlank(account)) {
                         resultFolder = "";
                     }
-                    
+
                     counts.compute(resultFolder, (k, v) -> v == null ? count : v + count);
                 }
-                
+
                 return counts.entrySet().stream()
                         .map(e -> new EmailCountsData(account, e.getKey(), e.getKey(), getFolderDisplayName(e.getKey()), e.getValue()))
                         .collect(Collectors.toList());

thunderbirdparser/ivy.xml

@@ -13,6 +13,7 @@
         <dependency conf="autopsy->default" org="org.apache.james" name="apache-mime4j-mbox-iterator" rev="0.8.0"/>
         <dependency conf="autopsy->default" org="com.googlecode.ez-vcard" name="ez-vcard" rev="0.10.5"/>
         <dependency conf="autopsy->default" org="com.github.mangstadt" name="vinnie" rev="2.0.2"/>
+        <dependency conf="autopsy->default" org="commons-io" name="commons-io" rev="2.5"/>
         <dependency org="com.google.guava" name="guava" rev="19.0"/>
         <dependency org="commons-validator" name="commons-validator" rev="1.6"/>
     </dependencies>

thunderbirdparser/nbproject/project.properties

@@ -10,6 +10,7 @@ file.reference.guava-19.0.jar=release/modules/ext/guava-19.0.jar
 file.reference.ez-vcard-0.10.5.jar=release/modules/ext/ez-vcard-0.10.5.jar
 file.reference.java-libpst-0.9.5-SNAPSHOT.jar=release/modules/ext/java-libpst-0.9.5-SNAPSHOT.jar
 file.reference.vinnie-2.0.2.jar=release/modules/ext/vinnie-2.0.2.jar
+file.reference.commons-io-2.5.jar=release/modules/ext/commons-io-2.5.jar
 javac.source=1.8
 javac.compilerargs=-Xlint -Xlint:-serial
 javadoc.reference.guava-19.0.jar=release/modules/ext/guava-19.0-javadoc.jar

thunderbirdparser/nbproject/project.xml

@@ -104,6 +104,10 @@
                 <runtime-relative-path>ext/commons-validator-1.6.jar</runtime-relative-path>
                 <binary-origin>release/modules/ext/commons-validator-1.6.jar</binary-origin>
             </class-path-extension>
+            <class-path-extension>
+                <runtime-relative-path>ext/commons-io-2.5.jar</runtime-relative-path>
+                <binary-origin>release/modules/ext/commons-io-2.5.jar</binary-origin>
+            </class-path-extension>
         </data>
     </configuration>
 </project>

thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java

@@ -137,7 +137,7 @@ class MimeJ4MessageParser implements AutoCloseable{
         email.setCc(getAddresses(msg.getCc()));
         email.setSubject(msg.getSubject());
         email.setSentDate(msg.getDate());
-        email.setLocalPath(FilenameUtils.getBaseName(localPath));
+        email.setLocalPath("\\" + FilenameUtils.getBaseName(localPath).replaceAll("\\\\", "/"));
         email.setMessageID(msg.getMessageId());
 
         Field field = msg.getHeader().getField("in-reply-to"); //NON-NLS

thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java

@@ -255,7 +255,7 @@ class PstParser  implements AutoCloseable{
         if (folder.hasSubfolders()) {
             List<PSTFolder> subFolders = folder.getSubFolders();
             for (PSTFolder subFolder : subFolders) {
-                String newpath = path + "/" + subFolder.getDisplayName();
+                String newpath = path + "\\" + subFolder.getDisplayName().replaceAll("\\\\", "/");
                 Iterable<EmailMessage> subIterable = getEmailMessageIterator(subFolder, newpath, fileID, wholeMsg);
                 if (subIterable == null) {
                     continue;