Merge branch 'develop' of github.com:sleuthkit/autopsy into solr-8-upgrade

Core/nbproject/project.xml

@@ -232,7 +232,8 @@
                     <build-prerequisite/>
                     <compile-dependency/>
                     <run-dependency>
-                        <specification-version>1.0</specification-version>
+                        <release-version>1</release-version>
+                        <specification-version>23</specification-version>
                     </run-dependency>
                 </dependency>
                 <dependency>

Core/test/qa-functional/src/org/sleuthkit/autopsy/README.txt

@@ -0,0 +1 @@
+Netbeans platform does not properly scope classloaders while running qa-functional test code.  The result is that NoClassDefError's occur in instances where an external jar (i.e. importing a class from common-io) is referenced in test code and the same external jar is referenced in multiple NBM's.  Importing from external jars in qa-functional should be avoided.  See jira issue 6954 for more information.

Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoAccountsTest.java

@@ -25,11 +25,10 @@ import java.nio.file.Paths;
 import junit.framework.Assert;
 import junit.framework.TestCase;
 import junit.framework.Test;
-import org.apache.commons.io.FileUtils;
 
 import org.netbeans.junit.NbModuleSuite;
-import org.openide.util.Exceptions;
 import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoAccount.CentralRepoAccountType;
+import org.sleuthkit.autopsy.coreutils.FileUtil;
 import org.sleuthkit.datamodel.Account;
 import org.sleuthkit.datamodel.InvalidAccountIDException;
 
@@ -95,7 +94,8 @@ public class CentralRepoAccountsTest extends TestCase {
         if (CentralRepository.isEnabled()) {
             CentralRepository.getInstance().shutdownConnections();
         }
-        FileUtils.deleteDirectory(testDirectory.toFile());
+
+        FileUtil.deleteDir(testDirectory.toFile());
     }
 
     public void testPredefinedAccountTypes() {

Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDatamodelTest.java

@@ -35,7 +35,6 @@ import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import junit.framework.Test;
 import junit.framework.TestCase;
-import org.apache.commons.io.FileUtils;
 import org.netbeans.junit.NbModuleSuite;
 import org.openide.util.Exceptions;
 import junit.framework.Assert;
@@ -47,6 +46,7 @@ import org.sleuthkit.datamodel.TskData;
 import static junit.framework.Assert.assertFalse;
 import static junit.framework.Assert.assertTrue;
 import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
+import org.sleuthkit.autopsy.coreutils.FileUtil;
 
 /**
  * Functional tests for the Central Repository data model.
@@ -100,8 +100,8 @@ public class CentralRepoDatamodelTest extends TestCase {
                 if (CentralRepository.isEnabled()) {
                     CentralRepository.getInstance().shutdownConnections();
                 }
-                FileUtils.deleteDirectory(testDirectory.toFile());
-            } catch (IOException | CentralRepoException ex) {
+                FileUtil.deleteDir(testDirectory.toFile());
+            } catch (CentralRepoException ex) {
                 Assert.fail(ex.getMessage());
             }
         }
@@ -194,8 +194,8 @@ public class CentralRepoDatamodelTest extends TestCase {
             if (CentralRepository.isEnabled()) {
                 CentralRepository.getInstance().shutdownConnections();
             }
-            FileUtils.deleteDirectory(testDirectory.toFile());
-        } catch (CentralRepoException | IOException ex) {
+            FileUtil.deleteDir(testDirectory.toFile());
+        } catch (CentralRepoException ex) {
             Exceptions.printStackTrace(ex);
             Assert.fail(ex.getMessage());
         }

Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoPersonasTest.java

@@ -28,10 +28,10 @@ import junit.framework.Assert;
 import static junit.framework.Assert.assertTrue;
 import junit.framework.TestCase;
 import junit.framework.Test;
-import org.apache.commons.io.FileUtils;
 
 import org.netbeans.junit.NbModuleSuite;
 import org.openide.util.Exceptions;
+import org.sleuthkit.autopsy.coreutils.FileUtil;
 import org.sleuthkit.datamodel.Account;
 import org.sleuthkit.datamodel.InvalidAccountIDException;
 import org.sleuthkit.datamodel.TskData;
@@ -257,12 +257,12 @@ public class CentralRepoPersonasTest  extends TestCase {
     // This function is run after every test, NOT after the entire collection of 
     // tests defined in the class are run.
     @Override
-    public void tearDown() throws CentralRepoException, IOException {
+    public void tearDown() throws CentralRepoException {
         // Close and delete the test case and central repo db
         if (CentralRepository.isEnabled()) {
             CentralRepository.getInstance().shutdownConnections();
         }
-        FileUtils.deleteDirectory(testDirectory.toFile());
+        FileUtil.deleteDir(testDirectory.toFile());
     }
     
     

Core/test/qa-functional/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseTestUtils.java

@@ -28,7 +28,6 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
-import org.apache.commons.io.FileUtils;
 import org.netbeans.junit.NbTestCase;
 import org.openide.util.Exceptions;
 import org.sleuthkit.autopsy.casemodule.Case;
@@ -63,6 +62,7 @@ import org.sleuthkit.autopsy.modules.vmextractor.VMExtractorIngestModuleFactory;
 import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepository;
 import org.sleuthkit.autopsy.centralrepository.datamodel.RdbmsCentralRepoFactory;
+import org.sleuthkit.autopsy.coreutils.FileUtil;
 import org.sleuthkit.autopsy.modules.pictureanalyzer.PictureAnalyzerIngestModuleFactory;
 
 /**
@@ -247,8 +247,8 @@ class InterCaseTestUtils {
                 if (CentralRepository.isEnabled()) {
                     CentralRepository.getInstance().shutdownConnections();
                 }
-                FileUtils.deleteDirectory(CENTRAL_REPO_DIRECTORY_PATH.toFile());
-            } catch (IOException | CentralRepoException ex) {
+                FileUtil.deleteDir(CENTRAL_REPO_DIRECTORY_PATH.toFile());
+            } catch (CentralRepoException ex) {
                 Exceptions.printStackTrace(ex);
                 Assert.fail(ex.getMessage());
             }

Experimental/nbproject/project.xml

@@ -134,7 +134,8 @@
                     <build-prerequisite/>
                     <compile-dependency/>
                     <run-dependency>
-                        <specification-version>1.0</specification-version>
+                        <release-version>1</release-version>
+                        <specification-version>23</specification-version>
                     </run-dependency>
                 </dependency>
                 <dependency>

KeywordSearch/nbproject/project.xml

@@ -118,7 +118,8 @@
                     <build-prerequisite/>
                     <compile-dependency/>
                     <run-dependency>
-                        <specification-version>1.0</specification-version>
+                        <release-version>1</release-version>
+                        <specification-version>23</specification-version>
                     </run-dependency>
                 </dependency>
                 <dependency>

Tika/manifest.mf

@@ -1,6 +1,7 @@
 Manifest-Version: 1.0
 AutoUpdate-Show-In-Client: true
-OpenIDE-Module: org.sleuthkit.autopsy.Tika
+OpenIDE-Module: org.sleuthkit.autopsy.Tika/1
+OpenIDE-Module-Implementation-Version: 1
 OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/Tika/Bundle.properties
-OpenIDE-Module-Specification-Version: 1.0
+OpenIDE-Module-Specification-Version: 23
 

ZookeeperNodeMigration/.gitignore

@@ -1,3 +1,3 @@
 /nbproject/private/
 /build/
-/dist/
+

ZookeeperNodeMigration/dist/README.TXT

@@ -0,0 +1,32 @@
+========================
+BUILD OUTPUT DESCRIPTION
+========================
+
+When you build an Java application project that has a main class, the IDE
+automatically copies all of the JAR
+files on the projects classpath to your projects dist/lib folder. The IDE
+also adds each of the JAR files to the Class-Path element in the application
+JAR files manifest file (MANIFEST.MF).
+
+To run the project from the command line, go to the dist folder and
+type the following:
+
+java -jar "ZookeeperNodeMigration.jar" 
+
+To distribute this project, zip up the dist folder (including the lib folder)
+and distribute the ZIP file.
+
+Notes:
+
+* If two JAR files on the project classpath have the same name, only the first
+JAR file is copied to the lib folder.
+* Only JAR files are copied to the lib folder.
+If the classpath contains other types of files or folders, these files (folders)
+are not copied.
+* If a library on the projects classpath also has a Class-Path element
+specified in the manifest,the content of the Class-Path element has to be on
+the projects runtime path.
+* To set a main class in a standard Java project, right-click the project node
+in the Projects window and choose Properties. Then click Run and enter the
+class name in the Main Class field. Alternatively, you can manually type the
+class name in the manifest Main-Class element.

thunderbirdparser/nbproject/project.xml

@@ -44,7 +44,8 @@
                     <build-prerequisite/>
                     <compile-dependency/>
                     <run-dependency>
-                        <specification-version>1.0</specification-version>
+                        <release-version>1</release-version>
+                        <specification-version>23</specification-version>
                     </run-dependency>
                 </dependency>
                 <dependency>

thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java

@@ -79,7 +79,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
     
     private static final int MBOX_SIZE_TO_SPLIT = 1048576000;
     private Case currentCase;
-
+  
     /**
      * Empty constructor.
      */
@@ -139,6 +139,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
         boolean isPstFile = PstParser.isPstFile(abstractFile);
         boolean isVcardFile = VcardParser.isVcardFile(abstractFile);
         
+        if (context.fileIngestIsCancelled()) {
+            return ProcessResult.OK;
+        }
+        
         if (isMbox || isEMLFile || isPstFile || isVcardFile  ) {
             try {
                 communicationArtifactsHelper = new CommunicationArtifactsHelper(currentCase.getSleuthkitCase(),
@@ -213,6 +217,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
                 Iterator<EmailMessage> pstMsgIterator = parser.getEmailMessageIterator();
                 if (pstMsgIterator != null) {
                     processEmails(parser.getPartialEmailMessages(), pstMsgIterator , abstractFile);
+                    if (context.fileIngestIsCancelled()) {
+                        return ProcessResult.OK;
+                    }
                 } else {
                     // sometimes parser returns ParseResult=OK but there are no messages
                     postErrorMessage(
@@ -321,6 +328,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
             }
 
             processMboxFile(file, abstractFile, emailFolder);
+            if (context.fileIngestIsCancelled()) {
+                return ProcessResult.OK;
+            }
             
             if (file.delete() == false) {
                 logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS
@@ -349,7 +359,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
                 if (splitFile.delete() == false) {
                     logger.log(Level.INFO, "Failed to delete temp file: {0}", splitFile); //NON-NLS
                 }
-
+                if (context.fileIngestIsCancelled()) {
+                    return ProcessResult.OK;
+                }
             }
         }                 
             
@@ -385,6 +397,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
         List<EmailMessage> emails = new ArrayList<>();
         if(emailIterator != null) {
             while(emailIterator.hasNext()) {
+                if (context.fileIngestIsCancelled()) {
+                    return;
+                }
                 EmailMessage emailMessage = emailIterator.next();
                 if(emailMessage != null) {
                     emails.add(emailMessage);
@@ -526,6 +541,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
 
         int msgCnt = 0;
         while(fullMessageIterator.hasNext()) {
+            if (context.fileIngestIsCancelled()) {
+                return;
+            }
+            
             EmailMessage current = fullMessageIterator.next();
             
             if(current == null) {
@@ -550,6 +569,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
 
         if (derivedFiles.isEmpty() == false) {
             for (AbstractFile derived : derivedFiles) {
+                if (context.fileIngestIsCancelled()) {
+                    return;
+                }
                 services.fireModuleContentEvent(new ModuleContentEvent(derived));
             }
         }
@@ -675,6 +697,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
         String senderAddress;
         senderAddressList.addAll(findEmailAddresess(from));
         
+        if (context.fileIngestIsCancelled()) {
+            return null;
+        }
+        
         AccountFileInstance senderAccountInstance = null;
 
         if (senderAddressList.size() == 1) {
@@ -690,13 +716,20 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
              logger.log(Level.WARNING, "Failed to find sender address, from  = {0}", from); //NON-NLS
         }
         
+        if (context.fileIngestIsCancelled()) {
+            return null;
+        }
+        
         List<String> recipientAddresses = new ArrayList<>();
         recipientAddresses.addAll(findEmailAddresess(to));
         recipientAddresses.addAll(findEmailAddresess(cc));
         recipientAddresses.addAll(findEmailAddresess(bcc));
         
         List<AccountFileInstance> recipientAccountInstances = new ArrayList<>();
-        recipientAddresses.forEach((addr) -> {
+        for (String addr : recipientAddresses) {
+            if (context.fileIngestIsCancelled()) {
+                return null;
+            }
             try {
                 AccountFileInstance recipientAccountInstance = 
                 currentCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, addr,
@@ -706,7 +739,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
             catch(TskCoreException ex) {
                 logger.log(Level.WARNING, "Failed to create account for email address  " + addr, ex); //NON-NLS
             }
-        });
+        }
                 
         addArtifactAttribute(headers, ATTRIBUTE_TYPE.TSK_HEADERS, bbattributes);
         addArtifactAttribute(from, ATTRIBUTE_TYPE.TSK_EMAIL_FROM, bbattributes);
@@ -731,12 +764,23 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
         
    
         try {
+            if (context.fileIngestIsCancelled()) {
+                return null;
+            }
             
             bbart = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG);
             bbart.addAttributes(bbattributes);
 
+            if (context.fileIngestIsCancelled()) {
+                return null;
+            }
+            
             // Add account relationships
             currentCase.getSleuthkitCase().getCommunicationsManager().addRelationships(senderAccountInstance, recipientAccountInstances, bbart,Relationship.Type.MESSAGE, dateL);
+
+            if (context.fileIngestIsCancelled()) {
+                return null;
+            }
             
             try {
                 // index the artifact for keyword search