diff --git a/Core/src/org/sleuthkit/autopsy/core/Installer.java b/Core/src/org/sleuthkit/autopsy/core/Installer.java index bca6eedfa6..0612729b61 100644 --- a/Core/src/org/sleuthkit/autopsy/core/Installer.java +++ b/Core/src/org/sleuthkit/autopsy/core/Installer.java @@ -204,7 +204,6 @@ public class Installer extends ModuleInstall { packageInstallers.add(org.sleuthkit.autopsy.centralrepository.eventlisteners.Installer.getDefault()); packageInstallers.add(org.sleuthkit.autopsy.healthmonitor.Installer.getDefault()); packageInstallers.add(org.sleuthkit.autopsy.casemodule.Installer.getDefault()); - packageInstallers.add(org.sleuthkit.autopsy.datasourcesummary.datamodel.Installer.getDefault()); /** * This is a temporary workaround for the following bug in Tika that diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/Installer.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/Installer.java deleted file mode 100644 index e9bd6c4060..0000000000 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/Installer.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2020 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.datasourcesummary.datamodel; - -import java.io.IOException; -import java.util.logging.Level; -import org.openide.modules.ModuleInstall; -import org.sleuthkit.autopsy.coreutils.Logger; - -/** - * Installer for data source summary that caches geolocation data. - */ -public final class Installer extends ModuleInstall { - - private static final Logger logger = Logger.getLogger(Installer.class.getName()); - private static final long serialVersionUID = 1L; - - private static Installer instance; - - public synchronized static Installer getDefault() { - if (instance == null) { - instance = new Installer(); - } - return instance; - } - - private Installer() { - } - - @Override - public void restored() { - WhereUsedSummary summary = WhereUsedSummary.getInstance(); - try { - summary.load(); - } catch (IOException ex) { - logger.log(Level.WARNING, "Unable to load geolocation summary data.", ex); - } - } -} diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/LatLngMap.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/LatLngMap.java index 6c647c5845..04fc52a8c4 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/LatLngMap.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/LatLngMap.java @@ -31,7 +31,7 @@ import org.sleuthkit.autopsy.geolocation.KdTree.XYZPoint; /** * Divides map into grid and places each grid square in separate index in a hashmap. */ -public class LatLngMap { +class LatLngMap { // radius of Earth in meters private static final double EARTH_RADIUS = 6371e3; @@ -60,7 +60,7 @@ public class LatLngMap { return Pair.of((int) (double) dPair.getLeft(), (int) (double) dPair.getRight()); }; - public LatLngMap(List pointsToAdd) { + LatLngMap(List pointsToAdd) { Map, List> latLngBuckets = pointsToAdd.stream() .collect(Collectors.groupingBy((pt) -> bucketCalculator.apply(pt))); @@ -69,7 +69,7 @@ public class LatLngMap { .collect(Collectors.toMap(p -> p.getKey(), p -> p.getValue())); } - public E findClosest(E point) { + E findClosest(E point) { Pair calculated = vectorDistance(point); int latBucket = (int) (double) calculated.getLeft(); int latBucket2 = Math.round(calculated.getLeft()) == latBucket ? latBucket - 1 : latBucket + 1; diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/WhereUsedSummary.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/WhereUsedSummary.java index 391205d5a3..7f5928a4ca 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/WhereUsedSummary.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/WhereUsedSummary.java @@ -25,6 +25,7 @@ import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -37,6 +38,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException; @@ -61,7 +63,7 @@ public class WhereUsedSummary implements DefaultArtifactUpdateGovernor { /** * A count of hits for a particular city. */ - public class CityCount { + public class CityRecordCount { private final CityRecord cityRecord; private final int count; @@ -73,7 +75,7 @@ public class WhereUsedSummary implements DefaultArtifactUpdateGovernor { * and location. * @param count The number of hits in proximity to that city. */ - CityCount(CityRecord cityRecord, int count) { + CityRecordCount(CityRecord cityRecord, int count) { this.cityRecord = cityRecord; this.count = count; } @@ -230,7 +232,7 @@ public class WhereUsedSummary implements DefaultArtifactUpdateGovernor { this.provider = provider; this.logger = logger; } - + /** * @return Returns all the geolocation artifact types. */ @@ -243,6 +245,73 @@ public class WhereUsedSummary implements DefaultArtifactUpdateGovernor { return GPS_ARTIFACT_TYPE_IDS; } + public static class CityCountsList { + + private final List counts; + private final int otherCount; + + public CityCountsList(List counts, int otherCount) { + this.counts = Collections.unmodifiableList(new ArrayList<>(counts)); + this.otherCount = otherCount; + } + + public List getCounts() { + return counts; + } + + public int getOtherCount() { + return otherCount; + } + } + + public static class CityData { + + private final CityCountsList mostCommon; + private final CityCountsList mostRecent; + private final Long mostRecentSeen; + + public CityData(CityCountsList mostCommon, CityCountsList mostRecent, Long mostRecentSeen) { + this.mostCommon = mostCommon; + this.mostRecent = mostRecent; + this.mostRecentSeen = mostRecentSeen; + } + + public CityCountsList getMostCommon() { + return mostCommon; + } + + public CityCountsList getMostRecent() { + return mostRecent; + } + + public Long getMostRecentSeen() { + return mostRecentSeen; + } + } + + private boolean greaterThanOrEqual(Long minTime, Long time) { + if ((minTime == null) || (time != null && time >= minTime)) { + return true; + } else { + return false; + } + } + + private static final Pair EMPTY_COUNT = Pair.of(0, 0); + + // left is total count, right is count within time range + private Pair getCounts(List points, Long minTime) { + + if (points == null) { + return EMPTY_COUNT; + } + + return points.stream().reduce( + EMPTY_COUNT, + (Waypoint w) -> Pair.of(1, greaterThanOrEqual(minTime, w.getTimestamp()) ? 1 : 0), + (pair1, pair2) -> Pair.of(pair1.getLeft() + pair2.getLeft(), pair1.getRight() + pair2.getRight())); + } + /** * Get this list of hits per city where the list is sorted descending by * number of found hits (i.e. most hits is first index). @@ -253,20 +322,65 @@ public class WhereUsedSummary implements DefaultArtifactUpdateGovernor { * @throws GeoLocationDataException * @throws InterruptedException */ - public List getCityCounts(DataSource dataSource) throws SleuthkitCaseProviderException, GeoLocationDataException, InterruptedException { + public CityData getCityCounts(DataSource dataSource, int daysCount, int maxCount) throws SleuthkitCaseProviderException, GeoLocationDataException, InterruptedException { if (this.latLngMap == null) { throw new IllegalStateException("City data hasn't been loaded"); } - - List dataSourcePoints = getPoints(dataSource); - Map cityCounts = getCounts(dataSourcePoints); - return cityCounts.entrySet().stream() - .map(e -> new CityCount(e.getKey(), e.getValue())) - .sorted((cityCount1, cityCount2) -> -Integer.compare(cityCount1.getCount(), cityCount2.getCount())) + List dataSourcePoints = getPoints(dataSource); + + Map> allCityPoints = new HashMap<>(); + List others = new ArrayList<>(); + Long mostRecent = null; + + for (Waypoint pt : dataSourcePoints) { + CityRecord city = latLngMap.findClosest(new CityRecord(null, null, pt.getLatitude(), pt.getLongitude())); + Long curTime = pt.getTimestamp(); + if (curTime != null && (mostRecent == null || curTime > mostRecent)) { + mostRecent = curTime; + } + + if (city == null) { + others.add(pt); + } else { + List cityPoints = allCityPoints.get(city); + if (cityPoints == null) { + cityPoints = new ArrayList<>(); + allCityPoints.put(city, cityPoints); + } + + cityPoints.add(pt); + } + } + + final Long mostRecentTime = mostRecent; + + // pair left is total count and right is count within range (or mostRecent is null) + Map> allCityCounts = allCityPoints.entrySet().stream() + .collect(Collectors.toMap((e) -> e.getKey(), (e) -> getCounts(e.getValue(), mostRecentTime))); + + List mostCommonCounts = allCityCounts.entrySet().stream() + .map(e -> new CityRecordCount(e.getKey(), e.getValue().getLeft())) + .sorted((a, b) -> -Integer.compare(a.getCount(), b.getCount())) + .limit(maxCount) .collect(Collectors.toList()); + + List mostRecentCounts = allCityCounts.entrySet().stream() + .map(e -> new CityRecordCount(e.getKey(), e.getValue().getRight())) + .sorted((a, b) -> -Integer.compare(a.getCount(), b.getCount())) + .limit(maxCount) + .collect(Collectors.toList()); + + Pair otherCounts = getCounts(others, mostRecentTime); + int otherMostCommonCount = otherCounts.getLeft(); + int otherMostRecentCount = otherCounts.getRight(); + + + return new CityData( + new CityCountsList(mostCommonCounts, otherMostCommonCount), + new CityCountsList(mostRecentCounts, otherMostRecentCount), + mostRecentTime); } - /** * Fetches all GPS data for the data source from the current case. @@ -313,24 +427,8 @@ public class WhereUsedSummary implements DefaultArtifactUpdateGovernor { * Pre-loads city data. */ public void load() throws IOException { - latLngMap = new LatLngMap(parseCsvLines(WhereUsedSummary.class.getResourceAsStream("worldcities.csv"), true)); - } - - private static CityRecord OTHER_RECORD = new CityRecord(Bundle.GeolocationSummary_cities_noRecordFound(), "", 0, 0); - - /** - * Determines closest city to each waypoint and returns a map of the city to - * the number of hits closest to that city. - * - * @param waypoints The waypoints. - * @return A map of city to the number of hits. - */ - private Map getCounts(List waypoints) { - Map toRet = waypoints.stream() - .map((point) -> latLngMap.findClosest(new CityRecord(null, null, point.getLatitude(), point.getLongitude()))) - .collect(Collectors.toMap(city -> city == null ? OTHER_RECORD : city, city -> 1, (count1, count2) -> count1 + count2)); - - return toRet; + latLngMap = new LatLngMap(parseCsvLines(WhereUsedSummary.class + .getResourceAsStream("worldcities.csv"), true)); } private static final int CITY_NAME_IDX = 0; diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/WhereUsedPanel.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/WhereUsedPanel.java index d1138c0f75..a31ab9af5a 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/WhereUsedPanel.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/WhereUsedPanel.java @@ -27,7 +27,7 @@ import org.openide.util.actions.CallableSystemAction; import org.openide.windows.TopComponent; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.datasourcesummary.datamodel.WhereUsedSummary; -import org.sleuthkit.autopsy.datasourcesummary.datamodel.WhereUsedSummary.CityCount; +import org.sleuthkit.autopsy.datasourcesummary.datamodel.WhereUsedSummary.CityRecordCount; import org.sleuthkit.autopsy.datasourcesummary.datamodel.WhereUsedSummary.CityRecord; import org.sleuthkit.autopsy.datasourcesummary.uiutils.CellModelTableCellRenderer.DefaultCellModel; import org.sleuthkit.autopsy.datasourcesummary.uiutils.DataFetchResult; @@ -75,20 +75,20 @@ public class WhereUsedPanel extends BaseDataSourceSummaryPanel { return String.format("%s, %s", record.getCityName(), record.getCountry()); } - private static final ColumnModel CITY_COL = new ColumnModel<>( + private static final ColumnModel CITY_COL = new ColumnModel<>( Bundle.WhereUsedPanel_cityColumn_title(), (cityCount) -> new DefaultCellModel(getCityName(cityCount.getCityRecord())), 300 ); - private static final ColumnModel COUNT_COL = new ColumnModel<>( + private static final ColumnModel COUNT_COL = new ColumnModel<>( Bundle.WhereUsedPanel_countColumn_title(), (cityCount) -> new DefaultCellModel(Integer.toString(cityCount.getCount())), 100 ); // table displaying city and number of hits for that city - private final JTablePanel cityCountsTable = JTablePanel.getJTablePanel(Arrays.asList(CITY_COL, COUNT_COL)) + private final JTablePanel cityCountsTable = JTablePanel.getJTablePanel(Arrays.asList(CITY_COL, COUNT_COL)) .setKeyFunction((cityCount) -> cityCount.getCityRecord()); // loadable components on this tab @@ -126,7 +126,7 @@ public class WhereUsedPanel extends BaseDataSourceSummaryPanel { initComponents(); } - private void handleData(DataFetchResult> result) { + private void handleData(DataFetchResult> result) { if (result != null && result.getResultType() == DataFetchResult.ResultType.SUCCESS && CollectionUtils.isNotEmpty(result.getData())) { viewInGeolocationBtn.setEnabled(true); }