CommonAttributeCaseSearchResults.java

Go to the documentation of this file.

/*
 *
 * Autopsy Forensic Browser
 *
 * Copyright 2018 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.commonfilesearch;

import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.logging.Level;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;

final public class CommonAttributeCaseSearchResults {

    private static final Logger LOGGER = Logger.getLogger(CommonAttributeCaseSearchResults.class.getName());

    // maps instance count to list of attribute values. 
    private final Map<String, Map<String, CommonAttributeValueList>> caseNameToDataSources;

    CommonAttributeCaseSearchResults(Map<String, Map<String, CommonAttributeValueList>> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType, Set<String> mimeTypesToFilterOn) {
        this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, resultType.getId(), mimeTypesToFilterOn);
    }

    CommonAttributeCaseSearchResults(Map<String, Map<String, CommonAttributeValueList>> metadata, int percentageThreshold) {
        this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, CorrelationAttributeInstance.FILES_TYPE_ID, new HashSet<>());
    }

    Map<String, CommonAttributeValueList> getAttributeValuesForCaseName(String caseName) {
        return this.caseNameToDataSources.get(caseName);
    }

    public Map<String, Map<String, CommonAttributeValueList>> getMetadata() {
        return Collections.unmodifiableMap(this.caseNameToDataSources);
    }

    private Map<String, Map<String, CommonAttributeValueList>> filterMetadata(Map<String, Map<String, CommonAttributeValueList>> metadata, int percentageThreshold, int resultTypeId, Set<String> mimeTypesToFilterOn) {
        try {
            final String currentCaseName;
            try {
                currentCaseName = Case.getCurrentCaseThrows().getDisplayName();
            } catch (NoCurrentCaseException ex) {
                throw new EamDbException("Unable to get current case while performing filtering", ex);
            }
            Map<String, CommonAttributeValueList> currentCaseDataSourceMap = metadata.get(currentCaseName);
            if (currentCaseDataSourceMap == null) {
                throw new EamDbException("No data for current case found in results, indicating there are no results and nothing will be filtered");
            }
            CorrelationAttributeInstance.Type attributeType = CorrelationAttributeInstance
                    .getDefaultCorrelationTypes()
                    .stream()
                    .filter(filterType -> filterType.getId() == resultTypeId)
                    .findFirst().get();
            //Call countUniqueDataSources once to reduce the number of DB queries needed to get the frequencyPercentage
            Double uniqueCaseDataSourceTuples = EamDb.getInstance().getCountUniqueDataSources().doubleValue();
            Map<String, Map<String, CommonAttributeValueList>> filteredCaseNameToDataSourcesTree = new HashMap<>();
            Map<String, CommonAttributeValue> valuesToKeepCurrentCase = getValuesToKeepFromCurrentCase(currentCaseDataSourceMap, attributeType, percentageThreshold, uniqueCaseDataSourceTuples, mimeTypesToFilterOn);
            for (Entry<String, Map<String, CommonAttributeValueList>> mapOfDataSources : Collections.unmodifiableMap(metadata).entrySet()) {
                if (!mapOfDataSources.getKey().equals(currentCaseName)) {
                    //rebuild the metadata structure with items from the current case substituted for their matches in other cases results we want to filter out removed
                    Map<String, CommonAttributeValueList> newTreeForCase = createTreeForCase(valuesToKeepCurrentCase, mapOfDataSources.getValue());
                    filteredCaseNameToDataSourcesTree.put(mapOfDataSources.getKey(), newTreeForCase);
                }
            }
            return filteredCaseNameToDataSourcesTree;
        } catch (EamDbException ex) {
            LOGGER.log(Level.INFO, "Unable to perform filtering returning unfiltered result set", ex);
            return metadata;
        }

    }

    private Map<String, CommonAttributeValue> getValuesToKeepFromCurrentCase(Map<String, CommonAttributeValueList> dataSourceToValueList, CorrelationAttributeInstance.Type attributeType, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples, Set<String> mimeTypesToFilterOn) throws EamDbException {
        Map<String, CommonAttributeValue> valuesToKeep = new HashMap<>();
        Set<String> valuesToRemove = new HashSet<>();
        for (Entry<String, CommonAttributeValueList> mapOfValueLists : Collections.unmodifiableMap(dataSourceToValueList).entrySet()) {
            for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataList()) {
                if (valuesToRemove.contains(value.getValue())) {
                    //do nothing this value will not be added
                } else if (filterValue(attributeType, value, maximumPercentageThreshold, uniqueCaseDataSourceTuples, mimeTypesToFilterOn)) {
                    valuesToRemove.add(value.getValue());
                } else {
                    valuesToKeep.put(value.getValue(), value);
                }
            }
        }
        return valuesToKeep;
    }

    private Map<String, CommonAttributeValueList> createTreeForCase(Map<String, CommonAttributeValue> valuesToKeepCurrentCase, Map<String, CommonAttributeValueList> dataSourceToValueList) throws EamDbException {
        Map<String, CommonAttributeValueList> treeForCase = new HashMap<>();
        for (Entry<String, CommonAttributeValueList> mapOfValueLists : Collections.unmodifiableMap(dataSourceToValueList).entrySet()) {
            for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataList()) {
                if (valuesToKeepCurrentCase.containsKey(value.getValue())) {
                    if (!treeForCase.containsKey(mapOfValueLists.getKey())) {
                        treeForCase.put(mapOfValueLists.getKey(), new CommonAttributeValueList());
                    }
                    treeForCase.get(mapOfValueLists.getKey()).addMetadataToList(valuesToKeepCurrentCase.get(value.getValue()));
                }
            }
        }
        return treeForCase;
    }

    private boolean filterValue(CorrelationAttributeInstance.Type attributeType, CommonAttributeValue value, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples, Set<String> mimeTypesToInclude) throws EamDbException {
        //Intracase common attribute searches will have been created with an empty mimeTypesToInclude list 
        //because when performing intra case search this filtering will have been done during the query of the case database 
        if (!mimeTypesToInclude.isEmpty()) { //only do the mime type filtering when mime types aren't empty
            for (AbstractCommonAttributeInstance commonAttr : value.getInstances()) {
                AbstractFile abstractFile = commonAttr.getAbstractFile();
                if (abstractFile != null) {
                    String mimeType = abstractFile.getMIMEType();
                    if (mimeType != null && !mimeTypesToInclude.contains(mimeType)) {
                        return true;
                    }
                }
            }
        }
        if (maximumPercentageThreshold != 0) {  //only do the frequency filtering when a max % was set
            try {
                Double uniqueTypeValueTuples = EamDb.getInstance().getCountUniqueCaseDataSourceTuplesHavingTypeValue(
                        attributeType, value.getValue()).doubleValue();
                Double commonalityPercentage = uniqueTypeValueTuples / uniqueCaseDataSourceTuples * 100;
                int frequencyPercentage = commonalityPercentage.intValue();
                if (frequencyPercentage > maximumPercentageThreshold) {
                    return true;
                }
            } catch (CorrelationAttributeNormalizationException ex) {
                LOGGER.log(Level.WARNING, "Unable to determine frequency percentage attribute - frequency filter may not be accurate for these results.", ex);
            }
        }
        return false;
    }
}