Autopsy  4.9.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
CommonAttributeCountSearchResults.java
Go to the documentation of this file.
1 /*
2  *
3  * Autopsy Forensic Browser
4  *
5  * Copyright 2018 Basis Technology Corp.
6  * Contact: carrier <at> sleuthkit <dot> org
7  *
8  * Licensed under the Apache License, Version 2.0 (the "License");
9  * you may not use this file except in compliance with the License.
10  * You may obtain a copy of the License at
11  *
12  * http://www.apache.org/licenses/LICENSE-2.0
13  *
14  * Unless required by applicable law or agreed to in writing, software
15  * distributed under the License is distributed on an "AS IS" BASIS,
16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17  * See the License for the specific language governing permissions and
18  * limitations under the License.
19  */
20 package org.sleuthkit.autopsy.commonfilesearch;
21 
22 import java.util.ArrayList;
23 import java.util.Collections;
24 import java.util.HashMap;
25 import java.util.HashSet;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.Map.Entry;
29 import java.util.Set;
30 import java.util.logging.Level;
36 import org.sleuthkit.datamodel.AbstractFile;
37 
43 
44  private static final Logger LOGGER = Logger.getLogger(CommonAttributeCountSearchResults.class.getName());
45 
46  // maps instance count to list of attribute values.
47  private final Map<Integer, CommonAttributeValueList> instanceCountToAttributeValues;
48  private final Set<String> mimeTypesToInclude;
49  private final int percentageThreshold;
50  private final int resultTypeId;
51 
64  CommonAttributeCountSearchResults(Map<Integer, CommonAttributeValueList> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType, Set<String> mimeTypesToFilterOn) {
65  //wrap in a new object in case any client code has used an unmodifiable collection
66  this.instanceCountToAttributeValues = new HashMap<>(metadata);
67  this.percentageThreshold = percentageThreshold;
68  this.resultTypeId = resultType.getId();
69  this.mimeTypesToInclude = mimeTypesToFilterOn;
70  }
71 
80  CommonAttributeCountSearchResults(Map<Integer, CommonAttributeValueList> metadata, int percentageThreshold) {
81  //wrap in a new object in case any client code has used an unmodifiable collection
82  this.instanceCountToAttributeValues = new HashMap<>(metadata);
83  this.percentageThreshold = percentageThreshold;
84  this.resultTypeId = CorrelationAttributeInstance.FILES_TYPE_ID;
85  this.mimeTypesToInclude = new HashSet<>(); //don't filter on mimetypes
86  }
87 
98  CommonAttributeValueList getAttributeValuesForInstanceCount(Integer instanceCount) {
99  return this.instanceCountToAttributeValues.get(instanceCount);
100  }
101 
109  public Map<Integer, CommonAttributeValueList> getMetadata() {
110  return Collections.unmodifiableMap(this.instanceCountToAttributeValues);
111  }
112 
118  public void filterMetadata() throws EamDbException {
119  filterMetadata(this.percentageThreshold);
120  }
121 
132  private void filterMetadata(int maximumPercentageThreshold) throws EamDbException {
135  .stream()
136  .filter(filterType -> filterType.getId() == this.resultTypeId)
137  .findFirst().get();
138 
139  EamDb eamDb = EamDb.getInstance();
140 
141  Map<Integer, List<CommonAttributeValue>> itemsToRemove = new HashMap<>();
142  //Call countUniqueDataSources once to reduce the number of DB queries needed to get
143  //the frequencyPercentage
144  Double uniqueCaseDataSourceTuples = eamDb.getCountUniqueDataSources().doubleValue();
145 
146  for (Entry<Integer, CommonAttributeValueList> listOfValues : Collections.unmodifiableMap(this.instanceCountToAttributeValues).entrySet()) {
147 
148  final Integer key = listOfValues.getKey();
149  final CommonAttributeValueList values = listOfValues.getValue();
150 
151  for (CommonAttributeValue value : values.getDelayedMetadataList()) { // Need the real metadata
152 
153  //Intracase common attribute searches will have been created with an empty mimeTypesToInclude list
154  //because when performing intra case search this filtering will have been done during the query of the case database
155  boolean mimeTypeToRemove = false; //allow code to be more efficient by not attempting to remove the same value multiple times
156  if (!mimeTypesToInclude.isEmpty()) { //only do the mime type filtering when mime types aren't empty
157  for (AbstractCommonAttributeInstance commonAttr : value.getInstances()) {
158  AbstractFile abstractFile = commonAttr.getAbstractFile();
159  if (abstractFile != null) {
160  String mimeType = commonAttr.getAbstractFile().getMIMEType();
161  if (mimeType != null && !mimeTypesToInclude.contains(mimeType)) {
162  if (itemsToRemove.containsKey(key)) {
163  itemsToRemove.get(key).add(value);
164  } else {
165  List<CommonAttributeValue> toRemove = new ArrayList<>();
166  toRemove.add(value);
167  itemsToRemove.put(key, toRemove);
168  }
169  //value will be removed as the mime type existed and was not in the set to be included
170  //because value is removed this value does not need to be checked further
171  mimeTypeToRemove = true;
172  break;
173  }
174  }
175  if (mimeTypeToRemove) {
176  break;
177  }
178  }
179  }
180  if (!mimeTypeToRemove && maximumPercentageThreshold != 0) { //only do the frequency filtering when a max % was set
181  try {
182  Double uniqueTypeValueTuples = eamDb.getCountUniqueCaseDataSourceTuplesHavingTypeValue(
183  attributeType, value.getValue()).doubleValue();
184  Double commonalityPercentage = uniqueTypeValueTuples / uniqueCaseDataSourceTuples * 100;
185  int frequencyPercentage = commonalityPercentage.intValue();
186  if (frequencyPercentage > maximumPercentageThreshold) {
187  if (itemsToRemove.containsKey(key)) {
188  itemsToRemove.get(key).add(value);
189  } else {
190  List<CommonAttributeValue> toRemove = new ArrayList<>();
191  toRemove.add(value);
192  itemsToRemove.put(key, toRemove);
193  }
194  }
196  LOGGER.log(Level.WARNING, "Unable to determine frequency percentage attribute - frequency filter may not be accurate for these results.", ex);
197  }
198  }
199  }
200  }
201  for (Entry<Integer, List<CommonAttributeValue>> valuesToRemove : itemsToRemove.entrySet()) {
202  final Integer key = valuesToRemove.getKey();
203  final List<CommonAttributeValue> values = valuesToRemove.getValue();
204  for (CommonAttributeValue value : values) {
205  final CommonAttributeValueList instanceCountValue = this.instanceCountToAttributeValues.get(key);
206  if (instanceCountValue != null) {
207  instanceCountValue.removeMetaData(value);
208  if (instanceCountValue.getDelayedMetadataList().isEmpty()) { // Check the real metadata
209  this.instanceCountToAttributeValues.remove(key);
210  }
211  }
212  }
213  }
214  }
215 
221  public int size() {
222 
223  int count = 0;
224  for (CommonAttributeValueList data : this.instanceCountToAttributeValues.values()) {
225  for (CommonAttributeValue md5 : data.getDelayedMetadataList()) {
226  count += md5.getInstanceCount();
227  }
228  }
229  return count;
230  }
231 }
Long getCountUniqueCaseDataSourceTuplesHavingTypeValue(CorrelationAttributeInstance.Type aType, String value)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124

Copyright © 2012-2018 Basis Technology. Generated on: Tue Dec 18 2018
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.