mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-12 16:06:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

working through city summary

Browse Source
This commit is contained in:
Greg DiCristofaro 2020-11-24 16:04:33 -05:00
parent aa6323559f
commit d458bc26d3
5 changed files with 135 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Core/src/org/sleuthkit/autopsy/core/Installer.java
View File

@ -204,7 +204,6 @@ public class Installer extends ModuleInstall {
packageInstallers.add(org.sleuthkit.autopsy.centralrepository.eventlisteners.Installer.getDefault());
packageInstallers.add(org.sleuthkit.autopsy.healthmonitor.Installer.getDefault());
packageInstallers.add(org.sleuthkit.autopsy.casemodule.Installer.getDefault());
packageInstallers.add(org.sleuthkit.autopsy.datasourcesummary.datamodel.Installer.getDefault());
/**
* This is a temporary workaround for the following bug in Tika that

55
Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/Installer.java
View File

@ -1,55 +0,0 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2020 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.datasourcesummary.datamodel;
import java.io.IOException;
import java.util.logging.Level;
import org.openide.modules.ModuleInstall;
import org.sleuthkit.autopsy.coreutils.Logger;
/**
* Installer for data source summary that caches geolocation data.
*/
public final class Installer extends ModuleInstall {
private static final Logger logger = Logger.getLogger(Installer.class.getName());
private static final long serialVersionUID = 1L;
private static Installer instance;
public synchronized static Installer getDefault() {
if (instance == null) {
instance = new Installer();
}
return instance;
}
private Installer() {
}
@Override
public void restored() {
WhereUsedSummary summary = WhereUsedSummary.getInstance();
try {
summary.load();
} catch (IOException ex) {
logger.log(Level.WARNING, "Unable to load geolocation summary data.", ex);
}
}
}

6
Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/LatLngMap.java
View File

@ -31,7 +31,7 @@ import org.sleuthkit.autopsy.geolocation.KdTree.XYZPoint;
/**
* Divides map into grid and places each grid square in separate index in a hashmap.
*/
public class LatLngMap<E extends KdTree.XYZPoint> {
class LatLngMap<E extends KdTree.XYZPoint> {
// radius of Earth in meters
private static final double EARTH_RADIUS = 6371e3;
@ -60,7 +60,7 @@ public class LatLngMap<E extends KdTree.XYZPoint> {
return Pair.of((int) (double) dPair.getLeft(), (int) (double) dPair.getRight());
};
public LatLngMap(List<E> pointsToAdd) {
LatLngMap(List<E> pointsToAdd) {
Map<Pair<Integer, Integer>, List<E>> latLngBuckets = pointsToAdd.stream()
.collect(Collectors.groupingBy((pt) -> bucketCalculator.apply(pt)));
@ -69,7 +69,7 @@ public class LatLngMap<E extends KdTree.XYZPoint> {
.collect(Collectors.toMap(p -> p.getKey(), p -> p.getValue()));
}
public E findClosest(E point) {
E findClosest(E point) {
Pair<Double, Double> calculated = vectorDistance(point);
int latBucket = (int) (double) calculated.getLeft();
int latBucket2 = Math.round(calculated.getLeft()) == latBucket ? latBucket - 1 : latBucket + 1;

156
Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/WhereUsedSummary.java
View File

@ -25,6 +25,7 @@ import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@ -37,6 +38,7 @@ import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException;
@ -61,7 +63,7 @@ public class WhereUsedSummary implements DefaultArtifactUpdateGovernor {
/**
* A count of hits for a particular city.
*/
public class CityCount {
public class CityRecordCount {
private final CityRecord cityRecord;
private final int count;
@ -73,7 +75,7 @@ public class WhereUsedSummary implements DefaultArtifactUpdateGovernor {
* and location.
* @param count The number of hits in proximity to that city.
*/
CityCount(CityRecord cityRecord, int count) {
CityRecordCount(CityRecord cityRecord, int count) {
this.cityRecord = cityRecord;
this.count = count;
}
@ -230,7 +232,7 @@ public class WhereUsedSummary implements DefaultArtifactUpdateGovernor {
this.provider = provider;
this.logger = logger;
}
/**
* @return Returns all the geolocation artifact types.
*/
@ -243,6 +245,73 @@ public class WhereUsedSummary implements DefaultArtifactUpdateGovernor {
return GPS_ARTIFACT_TYPE_IDS;
}
public static class CityCountsList {
private final List<CityRecordCount> counts;
private final int otherCount;
public CityCountsList(List<CityRecordCount> counts, int otherCount) {
this.counts = Collections.unmodifiableList(new ArrayList<>(counts));
this.otherCount = otherCount;
}
public List<CityRecordCount> getCounts() {
return counts;
}
public int getOtherCount() {
return otherCount;
}
}
public static class CityData {
private final CityCountsList mostCommon;
private final CityCountsList mostRecent;
private final Long mostRecentSeen;
public CityData(CityCountsList mostCommon, CityCountsList mostRecent, Long mostRecentSeen) {
this.mostCommon = mostCommon;
this.mostRecent = mostRecent;
this.mostRecentSeen = mostRecentSeen;
}
public CityCountsList getMostCommon() {
return mostCommon;
}
public CityCountsList getMostRecent() {
return mostRecent;
}
public Long getMostRecentSeen() {
return mostRecentSeen;
}
}
private boolean greaterThanOrEqual(Long minTime, Long time) {
if ((minTime == null) || (time != null && time >= minTime)) {
return true;
} else {
return false;
}
}
private static final Pair<Integer, Integer> EMPTY_COUNT = Pair.of(0, 0);
// left is total count, right is count within time range
private Pair<Integer, Integer> getCounts(List<Waypoint> points, Long minTime) {
if (points == null) {
return EMPTY_COUNT;
}
return points.stream().reduce(
EMPTY_COUNT,
(Waypoint w) -> Pair.of(1, greaterThanOrEqual(minTime, w.getTimestamp()) ? 1 : 0),
(pair1, pair2) -> Pair.of(pair1.getLeft() + pair2.getLeft(), pair1.getRight() + pair2.getRight()));
}
/**
* Get this list of hits per city where the list is sorted descending by
* number of found hits (i.e. most hits is first index).
@ -253,20 +322,65 @@ public class WhereUsedSummary implements DefaultArtifactUpdateGovernor {
* @throws GeoLocationDataException
* @throws InterruptedException
*/
public List<CityCount> getCityCounts(DataSource dataSource) throws SleuthkitCaseProviderException, GeoLocationDataException, InterruptedException {
public CityData getCityCounts(DataSource dataSource, int daysCount, int maxCount) throws SleuthkitCaseProviderException, GeoLocationDataException, InterruptedException {
if (this.latLngMap == null) {
throw new IllegalStateException("City data hasn't been loaded");
}
List<Waypoint> dataSourcePoints = getPoints(dataSource);
Map<CityRecord, Integer> cityCounts = getCounts(dataSourcePoints);
return cityCounts.entrySet().stream()
.map(e -> new CityCount(e.getKey(), e.getValue()))
.sorted((cityCount1, cityCount2) -> -Integer.compare(cityCount1.getCount(), cityCount2.getCount()))
List<Waypoint> dataSourcePoints = getPoints(dataSource);
Map<CityRecord, List<Waypoint>> allCityPoints = new HashMap<>();
List<Waypoint> others = new ArrayList<>();
Long mostRecent = null;
for (Waypoint pt : dataSourcePoints) {
CityRecord city = latLngMap.findClosest(new CityRecord(null, null, pt.getLatitude(), pt.getLongitude()));
Long curTime = pt.getTimestamp();
if (curTime != null && (mostRecent == null || curTime > mostRecent)) {
mostRecent = curTime;
}
if (city == null) {
others.add(pt);
} else {
List<Waypoint> cityPoints = allCityPoints.get(city);
if (cityPoints == null) {
cityPoints = new ArrayList<>();
allCityPoints.put(city, cityPoints);
}
cityPoints.add(pt);
}
}
final Long mostRecentTime = mostRecent;
// pair left is total count and right is count within range (or mostRecent is null)
Map<CityRecord, Pair<Integer, Integer>> allCityCounts = allCityPoints.entrySet().stream()
.collect(Collectors.toMap((e) -> e.getKey(), (e) -> getCounts(e.getValue(), mostRecentTime)));
List<CityRecordCount> mostCommonCounts = allCityCounts.entrySet().stream()
.map(e -> new CityRecordCount(e.getKey(), e.getValue().getLeft()))
.sorted((a, b) -> -Integer.compare(a.getCount(), b.getCount()))
.limit(maxCount)
.collect(Collectors.toList());
List<CityRecordCount> mostRecentCounts = allCityCounts.entrySet().stream()
.map(e -> new CityRecordCount(e.getKey(), e.getValue().getRight()))
.sorted((a, b) -> -Integer.compare(a.getCount(), b.getCount()))
.limit(maxCount)
.collect(Collectors.toList());
Pair<Integer, Integer> otherCounts = getCounts(others, mostRecentTime);
int otherMostCommonCount = otherCounts.getLeft();
int otherMostRecentCount = otherCounts.getRight();
return new CityData(
new CityCountsList(mostCommonCounts, otherMostCommonCount),
new CityCountsList(mostRecentCounts, otherMostRecentCount),
mostRecentTime);
}
/**
* Fetches all GPS data for the data source from the current case.
@ -313,24 +427,8 @@ public class WhereUsedSummary implements DefaultArtifactUpdateGovernor {
* Pre-loads city data.
*/
public void load() throws IOException {
latLngMap = new LatLngMap<CityRecord>(parseCsvLines(WhereUsedSummary.class.getResourceAsStream("worldcities.csv"), true));
}
private static CityRecord OTHER_RECORD = new CityRecord(Bundle.GeolocationSummary_cities_noRecordFound(), "", 0, 0);
/**
* Determines closest city to each waypoint and returns a map of the city to
* the number of hits closest to that city.
*
* @param waypoints The waypoints.
* @return A map of city to the number of hits.
*/
private Map<CityRecord, Integer> getCounts(List<Waypoint> waypoints) {
Map<CityRecord, Integer> toRet = waypoints.stream()
.map((point) -> latLngMap.findClosest(new CityRecord(null, null, point.getLatitude(), point.getLongitude())))
.collect(Collectors.toMap(city -> city == null ? OTHER_RECORD : city, city -> 1, (count1, count2) -> count1 + count2));
return toRet;
latLngMap = new LatLngMap<CityRecord>(parseCsvLines(WhereUsedSummary.class
.getResourceAsStream("worldcities.csv"), true));
}
private static final int CITY_NAME_IDX = 0;

10
Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/WhereUsedPanel.java
View File

@ -27,7 +27,7 @@ import org.openide.util.actions.CallableSystemAction;
import org.openide.windows.TopComponent;
import org.openide.windows.WindowManager;
import org.sleuthkit.autopsy.datasourcesummary.datamodel.WhereUsedSummary;
import org.sleuthkit.autopsy.datasourcesummary.datamodel.WhereUsedSummary.CityCount;
import org.sleuthkit.autopsy.datasourcesummary.datamodel.WhereUsedSummary.CityRecordCount;
import org.sleuthkit.autopsy.datasourcesummary.datamodel.WhereUsedSummary.CityRecord;
import org.sleuthkit.autopsy.datasourcesummary.uiutils.CellModelTableCellRenderer.DefaultCellModel;
import org.sleuthkit.autopsy.datasourcesummary.uiutils.DataFetchResult;
@ -75,20 +75,20 @@ public class WhereUsedPanel extends BaseDataSourceSummaryPanel {
return String.format("%s, %s", record.getCityName(), record.getCountry());
}
private static final ColumnModel<CityCount> CITY_COL = new ColumnModel<>(
private static final ColumnModel<CityRecordCount> CITY_COL = new ColumnModel<>(
Bundle.WhereUsedPanel_cityColumn_title(),
(cityCount) -> new DefaultCellModel(getCityName(cityCount.getCityRecord())),
300
);
private static final ColumnModel<CityCount> COUNT_COL = new ColumnModel<>(
private static final ColumnModel<CityRecordCount> COUNT_COL = new ColumnModel<>(
Bundle.WhereUsedPanel_countColumn_title(),
(cityCount) -> new DefaultCellModel(Integer.toString(cityCount.getCount())),
100
);
// table displaying city and number of hits for that city
private final JTablePanel<CityCount> cityCountsTable = JTablePanel.getJTablePanel(Arrays.asList(CITY_COL, COUNT_COL))
private final JTablePanel<CityRecordCount> cityCountsTable = JTablePanel.getJTablePanel(Arrays.asList(CITY_COL, COUNT_COL))
.setKeyFunction((cityCount) -> cityCount.getCityRecord());
// loadable components on this tab
@ -126,7 +126,7 @@ public class WhereUsedPanel extends BaseDataSourceSummaryPanel {
initComponents();
}
private void handleData(DataFetchResult<List<CityCount>> result) {
private void handleData(DataFetchResult<List<CityRecordCount>> result) {
if (result != null && result.getResultType() == DataFetchResult.ResultType.SUCCESS && CollectionUtils.isNotEmpty(result.getData())) {
viewInGeolocationBtn.setEnabled(true);
}