HashDbManager.java

Go to the documentation of this file.

/*
 * Autopsy Forensic Browser
 *
 * Copyright 2011-2018 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.modules.hashdatabase;

import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.beans.PropertyChangeSupport;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.logging.Level;
import java.util.stream.Stream;
import javax.swing.JFileChooser;
import javax.swing.JOptionPane;
import javax.swing.SwingWorker;
import javax.swing.filechooser.FileNameExtensionFilter;
import org.apache.commons.io.FilenameUtils;
import org.netbeans.api.progress.ProgressHandle;
import org.openide.util.NbBundle;
import org.openide.util.NbBundle.Messages;
import org.openide.windows.WindowManager;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance;
import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoException;
import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoFileInstance;
import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoFileSet;
import org.sleuthkit.autopsy.core.RuntimeProperties;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.ingest.IngestManager;
import org.sleuthkit.autopsy.modules.hashdatabase.HashLookupSettings.HashDbInfo;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.HashEntry;
import org.sleuthkit.datamodel.HashHitInfo;
import org.sleuthkit.datamodel.SleuthkitJNI;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepository;
import org.sleuthkit.autopsy.modules.hashdatabase.HashDbManager.HashDb.KnownFilesType;

public class HashDbManager implements PropertyChangeListener {

    private static final String HASH_DATABASE_FILE_EXTENSON = "kdb"; //NON-NLS
    private static HashDbManager instance = null;
    private List<HashDb> hashSets = new ArrayList<>();
    private Set<String> hashSetNames = new HashSet<>();
    private Set<String> hashSetPaths = new HashSet<>();
    PropertyChangeSupport changeSupport = new PropertyChangeSupport(HashDbManager.class);
    private static final Logger logger = Logger.getLogger(HashDbManager.class.getName());
    private boolean allDatabasesLoadedCorrectly = false;

    public enum SetEvt {

        DB_ADDED, DB_DELETED, DB_INDEXED
    };

    public static synchronized HashDbManager getInstance() {
        if (instance == null) {
            instance = new HashDbManager();
        }
        return instance;
    }

    public synchronized void addPropertyChangeListener(PropertyChangeListener listener) {
        changeSupport.addPropertyChangeListener(listener);
    }

    public synchronized void removePropertyChangeListener(PropertyChangeListener listener) {
        changeSupport.removePropertyChangeListener(listener);
    }

    synchronized boolean verifyAllDatabasesLoadedCorrectly() {
        return allDatabasesLoadedCorrectly;
    }

    private HashDbManager() {
        loadHashsetsConfiguration();
    }

    static String getHashDatabaseFileExtension() {
        return HASH_DATABASE_FILE_EXTENSON;
    }

    public class HashDbManagerException extends Exception {

        private static final long serialVersionUID = 1L;

        private HashDbManagerException(String message) {
            super(message);
        }

        private HashDbManagerException(String message, Throwable exception) {
            super(message, exception);
        }
    }

    public synchronized HashDb addExistingHashDatabase(String hashSetName, String path, boolean searchDuringIngest, boolean sendIngestMessages, HashDb.KnownFilesType knownFilesType) throws HashDbManagerException {
        HashDb hashDb = null;
        hashDb = this.addExistingHashDatabaseNoSave(hashSetName, path, searchDuringIngest, sendIngestMessages, knownFilesType);
        this.save();
        return hashDb;
    }

    synchronized HashDb addExistingHashDatabaseNoSave(String hashSetName, String path, boolean searchDuringIngest, boolean sendIngestMessages, HashDb.KnownFilesType knownFilesType) throws HashDbManagerException {
        HashDb hashDb = null;
        try {
            if (!new File(path).exists()) {
                throw new HashDbManagerException(NbBundle.getMessage(HashDbManager.class, "HashDbManager.hashDbDoesNotExistExceptionMsg", path));
            }

            if (hashSetPaths.contains(path)) {
                throw new HashDbManagerException(NbBundle.getMessage(HashDbManager.class, "HashDbManager.hashDbAlreadyAddedExceptionMsg", path));
            }

            if (hashSetNames.contains(hashSetName)) {
                throw new HashDbManagerException(NbBundle.getMessage(HashDbManager.class, "HashDbManager.duplicateHashSetNameExceptionMsg", hashSetName));
            }

            hashDb = addHashDatabase(SleuthkitJNI.openHashDatabase(path), hashSetName, searchDuringIngest, sendIngestMessages, knownFilesType);
        } catch (TskCoreException ex) {
            throw new HashDbManagerException(ex.getMessage());
        }
        return hashDb;
    }

    public synchronized HashDb addNewHashDatabase(String hashSetName, String path, boolean searchDuringIngest, boolean sendIngestMessages,
            HashDb.KnownFilesType knownFilesType) throws HashDbManagerException {

        HashDb hashDb = null;
        hashDb = this.addNewHashDatabaseNoSave(hashSetName, path, searchDuringIngest, sendIngestMessages, knownFilesType);

        this.save();

        return hashDb;
    }

    public synchronized HashDb addNewHashDatabaseNoSave(String hashSetName, String path, boolean searchDuringIngest, boolean sendIngestMessages,
            HashDb.KnownFilesType knownFilesType) throws HashDbManagerException {
        HashDb hashDb = null;
        try {
            File file = new File(path);
            if (file.exists()) {
                throw new HashDbManagerException(NbBundle.getMessage(HashDbManager.class, "HashDbManager.hashDbFileExistsExceptionMsg", path));
            }
            if (!FilenameUtils.getExtension(file.getName()).equalsIgnoreCase(HASH_DATABASE_FILE_EXTENSON)) {
                throw new HashDbManagerException(NbBundle.getMessage(HashDbManager.class, "HashDbManager.illegalHashDbFileNameExtensionMsg",
                        getHashDatabaseFileExtension()));
            }

            if (hashSetPaths.contains(path)) {
                throw new HashDbManagerException(NbBundle.getMessage(HashDbManager.class, "HashDbManager.hashDbAlreadyAddedExceptionMsg", path));
            }

            if (hashSetNames.contains(hashSetName)) {
                throw new HashDbManagerException(NbBundle.getMessage(HashDbManager.class, "HashDbManager.duplicateHashSetNameExceptionMsg", hashSetName));
            }

            hashDb = addHashDatabase(SleuthkitJNI.createHashDatabase(path), hashSetName, searchDuringIngest, sendIngestMessages, knownFilesType);
        } catch (TskCoreException ex) {
            throw new HashDbManagerException(ex.getMessage());
        }
        return hashDb;
    }

    private SleuthkitHashSet addHashDatabase(int handle, String hashSetName, boolean searchDuringIngest, boolean sendIngestMessages, HashDb.KnownFilesType knownFilesType) throws TskCoreException {
        // Wrap an object around the handle.
        SleuthkitHashSet hashDb = new SleuthkitHashSet(handle, hashSetName, searchDuringIngest, sendIngestMessages, knownFilesType);

        // Get the indentity data before updating the collections since the 
        // accessor methods may throw. 
        String databasePath = hashDb.getDatabasePath();
        String indexPath = hashDb.getIndexPath();

        // Update the collections used to ensure that hash set names are unique 
        // and the same database is not added to the configuration more than once.
        hashSetNames.add(hashDb.getHashSetName());
        if (!databasePath.equals("None")) { //NON-NLS
            hashSetPaths.add(databasePath);
        }
        if (!indexPath.equals("None")) { //NON-NLS
            hashSetPaths.add(indexPath);
        }

        // Add the hash database to the collection
        hashSets.add(hashDb);

        // Let any external listeners know that there's a new set   
        try {
            changeSupport.firePropertyChange(SetEvt.DB_ADDED.toString(), null, hashSetName);
        } catch (Exception e) {
            logger.log(Level.SEVERE, "HashDbManager listener threw exception", e); //NON-NLS
            MessageNotifyUtil.Notify.show(
                    NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErr"),
                    NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErrorListeningToUpdatesMsg"),
                    MessageNotifyUtil.MessageType.ERROR);
        }
        return hashDb;
    }

    CentralRepoHashSet addExistingCentralRepoHashSet(String hashSetName, String version, int referenceSetID,
            boolean searchDuringIngest, boolean sendIngestMessages, HashDb.KnownFilesType knownFilesType,
            boolean readOnly) throws TskCoreException {

        if (!CentralRepository.isEnabled()) {
            throw new TskCoreException("Could not load central repository hash set " + hashSetName + " - central repository is not enabled");
        }

        CentralRepoHashSet db = new CentralRepoHashSet(hashSetName, version, referenceSetID, searchDuringIngest,
                sendIngestMessages, knownFilesType, readOnly);

        if (!db.isValid()) {
            throw new TskCoreException("Error finding hash set " + hashSetName + " in central repository");
        }

        // Add the hash database to the collection
        hashSets.add(db);

        // Let any external listeners know that there's a new set   
        try {
            changeSupport.firePropertyChange(SetEvt.DB_ADDED.toString(), null, hashSetName);
        } catch (Exception e) {
            logger.log(Level.SEVERE, "HashDbManager listener threw exception", e); //NON-NLS
            MessageNotifyUtil.Notify.show(
                    NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErr"),
                    NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErrorListeningToUpdatesMsg"),
                    MessageNotifyUtil.MessageType.ERROR);
        }
        return db;

    }

    synchronized void indexHashDatabase(SleuthkitHashSet hashDb) {
        hashDb.addPropertyChangeListener(this);
        HashDbIndexer creator = new HashDbIndexer(hashDb);
        creator.execute();
    }

    @Override
    public void propertyChange(PropertyChangeEvent event) {
        if (event.getPropertyName().equals(SleuthkitHashSet.Event.INDEXING_DONE.name())) {
            SleuthkitHashSet hashDb = (SleuthkitHashSet) event.getNewValue();
            if (null != hashDb) {
                try {
                    String indexPath = hashDb.getIndexPath();
                    if (!indexPath.equals("None")) { //NON-NLS
                        hashSetPaths.add(indexPath);
                    }
                } catch (TskCoreException ex) {
                    Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error getting index path of " + hashDb.getHashSetName() + " hash set after indexing", ex); //NON-NLS
                }
            }
        }
    }

    public synchronized void removeHashDatabase(HashDb hashDb) throws HashDbManagerException {
        this.removeHashDatabaseNoSave(hashDb);
        this.save();
    }

    public synchronized void removeHashDatabaseNoSave(HashDb hashDb) throws HashDbManagerException {
        // Don't remove a database if ingest is running
        boolean ingestIsRunning = IngestManager.getInstance().isIngestRunning();
        if (ingestIsRunning) {
            throw new HashDbManagerException(NbBundle.getMessage(this.getClass(), "HashDbManager.ingestRunningExceptionMsg"));
        }
        // Remove the database from whichever hash set list it occupies,
        // and remove its hash set name from the hash set used to ensure unique
        // hash set names are used, before undertaking These operations will succeed and constitute
        // a mostly effective removal, even if the subsequent operations fail.
        String hashSetName = hashDb.getHashSetName();
        hashSetNames.remove(hashSetName);
        hashSets.remove(hashDb);

        // Now undertake the operations that could throw.
        // Indexing is only relevanet for sleuthkit hashsets
        if (hashDb instanceof SleuthkitHashSet) {
            SleuthkitHashSet hashDatabase = (SleuthkitHashSet) hashDb;
            try {
                if (hashDatabase.hasIndex()) {
                    hashSetPaths.remove(hashDatabase.getIndexPath());
                }
            } catch (TskCoreException ex) {
                Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error getting index path of " + hashDatabase.getHashSetName() + " hash set when removing the hash set", ex); //NON-NLS
            }

            try {
                if (!hashDatabase.hasIndexOnly()) {
                    hashSetPaths.remove(hashDatabase.getDatabasePath());
                }
            } catch (TskCoreException ex) {
                Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error getting hash set path of " + hashDatabase.getHashSetName() + " hash set when removing the hash set", ex); //NON-NLS
            }

            try {
                hashDatabase.close();
            } catch (TskCoreException ex) {
                Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error closing " + hashDb.getHashSetName() + " hash set when removing the hash set", ex); //NON-NLS
            }
        }

        // Let any external listeners know that a set has been deleted
        try {
            changeSupport.firePropertyChange(SetEvt.DB_DELETED.toString(), null, hashSetName);
        } catch (Exception e) {
            logger.log(Level.SEVERE, "HashDbManager listener threw exception", e); //NON-NLS
            MessageNotifyUtil.Notify.show(
                    NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErr"),
                    NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErrorListeningToUpdatesMsg"),
                    MessageNotifyUtil.MessageType.ERROR);
        }
    }

    void save() throws HashDbManagerException {
        try {
            if (!HashLookupSettings.writeSettings(new HashLookupSettings(HashLookupSettings.convertHashSetList(this.hashSets)))) {
                throw new HashDbManagerException(NbBundle.getMessage(this.getClass(), "HashDbManager.saveErrorExceptionMsg"));
            }
        } catch (HashLookupSettings.HashLookupSettingsException ex) {
            throw new HashDbManagerException(NbBundle.getMessage(this.getClass(), "HashDbManager.saveErrorExceptionMsg"));
        }
    }

    public synchronized List<HashDb> getAllHashSets() {
        try {
            updateHashSetsFromCentralRepository();
        } catch (TskCoreException ex) {
            Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error loading central repository hash sets", ex); //NON-NLS
        }

        List<HashDb> hashDbs = new ArrayList<>();
        hashDbs.addAll(this.hashSets);
        return hashDbs;
    }

    public synchronized List<HashDb> getKnownFileHashSets() {
        List<HashDb> hashDbs = new ArrayList<>();
        try {
            updateHashSetsFromCentralRepository();
        } catch (TskCoreException ex) {
            Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error loading central repository hash sets", ex); //NON-NLS
        }
        this.hashSets.stream().filter((db) -> (db.getKnownFilesType() == HashDb.KnownFilesType.KNOWN)).forEach((db) -> {
            hashDbs.add(db);
        });
        return hashDbs;
    }

    public synchronized List<HashDb> getKnownBadFileHashSets() {
        List<HashDb> hashDbs = new ArrayList<>();
        try {
            updateHashSetsFromCentralRepository();
        } catch (TskCoreException ex) {
            Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error loading central repository hash sets", ex); //NON-NLS
        }
        this.hashSets.stream().filter((db) -> (db.getKnownFilesType() == HashDb.KnownFilesType.KNOWN_BAD)).forEach((db) -> {
            hashDbs.add(db);
        });
        return hashDbs;
    }

    public synchronized List<HashDb> getUpdateableHashSets() {
        return getUpdateableHashSets(this.hashSets);
    }

    private List<HashDb> getUpdateableHashSets(List<HashDb> hashDbs) {
        ArrayList<HashDb> updateableDbs = new ArrayList<>();
        try {
            updateHashSetsFromCentralRepository();
        } catch (TskCoreException ex) {
            Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error loading central repository hash sets", ex); //NON-NLS
        }
        for (HashDb db : hashDbs) {
            try {
                if (db.isUpdateable()) {
                    updateableDbs.add(db);
                }
            } catch (TskCoreException ex) {
                Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error checking updateable status of " + db.getHashSetName() + " hash set", ex); //NON-NLS
            }
        }
        return updateableDbs;
    }

    private List<HashDbInfo> getCentralRepoHashSetsFromDatabase() {
        List<HashDbInfo> crHashSets = new ArrayList<>();
        if (CentralRepository.isEnabled()) {
            try {
                List<CentralRepoFileSet> crSets = CentralRepository.getInstance().getAllReferenceSets(CentralRepository.getInstance().getCorrelationTypeById(CorrelationAttributeInstance.FILES_TYPE_ID));
                for (CentralRepoFileSet globalSet : crSets) {

                    // Defaults for fields not stored in the central repository:
                    //   searchDuringIngest: false
                    //   sendIngestMessages: true if the hash set is notable
                    boolean sendIngestMessages = KnownFilesType.fromFileKnown(globalSet.getFileKnownStatus()).equals(HashDb.KnownFilesType.KNOWN_BAD);
                    crHashSets.add(new HashDbInfo(globalSet.getSetName(), globalSet.getVersion(),
                            globalSet.getGlobalSetID(), KnownFilesType.fromFileKnown(globalSet.getFileKnownStatus()), globalSet.isReadOnly(), false, sendIngestMessages));
                }
            } catch (CentralRepoException ex) {
                Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error loading central repository hash sets", ex); //NON-NLS
            }
        }
        return crHashSets;
    }

    public synchronized void loadLastSavedConfiguration() {
        closeHashDatabases(this.hashSets);
        hashSetNames.clear();
        hashSetPaths.clear();

        loadHashsetsConfiguration();
    }

    private void closeHashDatabases(List<HashDb> hashDatabases) {
        for (HashDb database : hashDatabases) {
            if (database instanceof SleuthkitHashSet) {
                try {
                    ((SleuthkitHashSet) database).close();
                } catch (TskCoreException ex) {
                    Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error closing " + database.getHashSetName() + " hash set", ex); //NON-NLS
                }
            }
        }
        hashDatabases.clear();
    }

    private void loadHashsetsConfiguration() {
        try {
            HashLookupSettings settings = HashLookupSettings.readSettings();
            this.configureSettings(settings);
        } catch (HashLookupSettings.HashLookupSettingsException ex) {
            Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Could not read Hash lookup settings from disk.", ex);
        }
    }

    @Messages({"# {0} - hash set name", "HashDbManager.noDbPath.message=Couldn't get valid hash set path for: {0}",
        "HashDbManager.centralRepoLoadError.message=Error loading central repository hash sets"})
    private void configureSettings(HashLookupSettings settings) {
        allDatabasesLoadedCorrectly = true;
        List<HashDbInfo> hashDbInfoList = settings.getHashDbInfo();
        for (HashDbInfo hashDbInfo : hashDbInfoList) {
            try {
                if (hashDbInfo.isFileDatabaseType()) {
                    String dbPath = this.getValidFilePath(hashDbInfo.getHashSetName(), hashDbInfo.getPath());
                    if (dbPath != null) {
                        addHashDatabase(SleuthkitJNI.openHashDatabase(dbPath), hashDbInfo.getHashSetName(), hashDbInfo.getSearchDuringIngest(), hashDbInfo.getSendIngestMessages(), hashDbInfo.getKnownFilesType());
                    } else {
                        logger.log(Level.WARNING, Bundle.HashDbManager_noDbPath_message(hashDbInfo.getHashSetName()));
                        allDatabasesLoadedCorrectly = false;
                    }
                } else {
                    if (CentralRepository.isEnabled()) {
                        addExistingCentralRepoHashSet(hashDbInfo.getHashSetName(), hashDbInfo.getVersion(),
                                hashDbInfo.getReferenceSetID(),
                                hashDbInfo.getSearchDuringIngest(), hashDbInfo.getSendIngestMessages(),
                                hashDbInfo.getKnownFilesType(), hashDbInfo.isReadOnly());
                    }
                }
            } catch (TskCoreException ex) {
                Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error opening hash set", ex); //NON-NLS
                JOptionPane.showMessageDialog(WindowManager.getDefault().getMainWindow(),
                        NbBundle.getMessage(this.getClass(),
                                "HashDbManager.unableToOpenHashDbMsg", hashDbInfo.getHashSetName()),
                        NbBundle.getMessage(this.getClass(), "HashDbManager.openHashDbErr"),
                        JOptionPane.ERROR_MESSAGE);
                allDatabasesLoadedCorrectly = false;
            }
        }

        if (CentralRepository.isEnabled()) {
            try {
                updateHashSetsFromCentralRepository();
            } catch (TskCoreException ex) {
                Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error opening hash set", ex); //NON-NLS

                JOptionPane.showMessageDialog(WindowManager.getDefault().getMainWindow(),
                        Bundle.HashDbManager_centralRepoLoadError_message(),
                        NbBundle.getMessage(this.getClass(), "HashDbManager.openHashDbErr"),
                        JOptionPane.ERROR_MESSAGE);
                allDatabasesLoadedCorrectly = false;
            }
        }

        /*
         * NOTE: When RuntimeProperties.coreComponentsAreActive() is "false", I
         * don't think we should overwrite hash db settings file because we were
         * unable to load a database. The user should have to fix the issue or
         * remove the database from settings. Overwiting the settings
         * effectively removes the database from HashLookupSettings and the user
         * may not know about this because the dialogs are not being displayed.
         * The next time user starts Autopsy, HashDB will load without errors
         * and the user may think that the problem was solved.
         */
        if (!allDatabasesLoadedCorrectly && RuntimeProperties.runningWithGUI()) {
            try {
                HashLookupSettings.writeSettings(new HashLookupSettings(HashLookupSettings.convertHashSetList(this.hashSets)));
                allDatabasesLoadedCorrectly = true;
            } catch (HashLookupSettings.HashLookupSettingsException ex) {
                allDatabasesLoadedCorrectly = false;
                logger.log(Level.SEVERE, "Could not overwrite hash set settings.", ex);
            }
        }
    }

    private void updateHashSetsFromCentralRepository() throws TskCoreException {
        if (CentralRepository.isEnabled()) {
            List<HashDbInfo> crHashDbInfoList = getCentralRepoHashSetsFromDatabase();
            for (HashDbInfo hashDbInfo : crHashDbInfoList) {
                if (hashDbInfoIsNew(hashDbInfo)) {
                    addExistingCentralRepoHashSet(hashDbInfo.getHashSetName(), hashDbInfo.getVersion(),
                            hashDbInfo.getReferenceSetID(),
                            hashDbInfo.getSearchDuringIngest(), hashDbInfo.getSendIngestMessages(), hashDbInfo.getKnownFilesType(),
                            hashDbInfo.isReadOnly());
                }
            }
        }
    }

    private boolean hashDbInfoIsNew(HashDbInfo dbInfo) {
        for (HashDb db : this.hashSets) {
            if (dbInfo.matches(db)) {
                return false;
            }
        }
        return true;
    }

    private String getValidFilePath(String hashSetName, String configuredPath) {
        // Check the configured path.
        File database = new File(configuredPath);
        if (database.exists()) {
            return configuredPath;
        }

        // Give the user an opportunity to find the desired file.
        String newPath = null;
        if (RuntimeProperties.runningWithGUI()
                && JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(),
                        NbBundle.getMessage(this.getClass(), "HashDbManager.dlgMsg.dbNotFoundAtLoc",
                                hashSetName, configuredPath),
                        NbBundle.getMessage(this.getClass(), "HashDbManager.dlgTitle.MissingDb"),
                        JOptionPane.YES_NO_OPTION) == JOptionPane.YES_OPTION) {
            newPath = searchForFile();
            if (null != newPath && !newPath.isEmpty()) {
                database = new File(newPath);
                if (!database.exists()) {
                    newPath = null;
                }
            }
        }
        return newPath;
    }

    private String searchForFile() {
        String filePath = null;
        JFileChooser fc = new JFileChooser();
        fc.setDragEnabled(false);
        fc.setFileSelectionMode(JFileChooser.FILES_ONLY);
        String[] EXTENSION = new String[]{"txt", "idx", "hash", "Hash", "kdb"}; //NON-NLS
        FileNameExtensionFilter filter = new FileNameExtensionFilter(
                NbBundle.getMessage(this.getClass(), "HashDbManager.fileNameExtensionFilter.title"), EXTENSION);
        fc.setFileFilter(filter);
        fc.setMultiSelectionEnabled(false);
        if (fc.showOpenDialog(null) == JFileChooser.APPROVE_OPTION) {
            File f = fc.getSelectedFile();
            try {
                filePath = f.getCanonicalPath();
            } catch (IOException ex) {
                Logger.getLogger(HashDbManager.class.getName()).log(Level.WARNING, "Couldn't get selected file path", ex); //NON-NLS
            }
        }
        return filePath;
    }

    public static abstract class HashDb {

        @Messages({
            "HashDbManager.noChange.text=No Change",
            "HashDbManager.known.text=Known",
            "HashDbManager.knownBad.text=Notable"
        })
        public enum KnownFilesType {

            KNOWN(Bundle.HashDbManager_known_text(), TskData.FileKnown.KNOWN, false, false),
            KNOWN_BAD(Bundle.HashDbManager_knownBad_text(), TskData.FileKnown.BAD, true, true),
            NO_CHANGE(Bundle.HashDbManager_noChange_text(), TskData.FileKnown.UNKNOWN, true, false);

            private final String displayName;
            private final TskData.FileKnown fileKnown;
            private final boolean allowSendInboxMessages;
            private final boolean defaultSendInboxMessages;

            KnownFilesType(String displayName, TskData.FileKnown fileKnown, boolean allowSendInboxMessages, boolean defaultSendInboxMessages) {
                this.displayName = displayName;
                this.fileKnown = fileKnown;
                this.allowSendInboxMessages = allowSendInboxMessages;
                this.defaultSendInboxMessages = defaultSendInboxMessages;
            }

            boolean isInboxMessagesAllowed() {
                return allowSendInboxMessages;
            }

            boolean isDefaultInboxMessages() {
                return defaultSendInboxMessages;
            }

            public String getDisplayName() {
                return this.displayName;
            }

            TskData.FileKnown getFileKnown() {
                return this.fileKnown;
            }

            static KnownFilesType fromFileKnown(TskData.FileKnown fileKnown) {
                if (fileKnown == null) {
                    return null;
                }

                return Stream.of(KnownFilesType.values())
                        .filter((type) -> type.getFileKnown() == fileKnown)
                        .findFirst()
                        .orElseThrow(() -> new IllegalArgumentException("Unknown TskData.FileKnown type: " + fileKnown));
            }
        }

        public enum Event {

            INDEXING_DONE
        }

        public abstract String getHashSetName();

        abstract String getDisplayName();

        public abstract String getDatabasePath() throws TskCoreException;

        public abstract HashDb.KnownFilesType getKnownFilesType();

        public abstract boolean getSearchDuringIngest();

        abstract void setSearchDuringIngest(boolean useForIngest);

        public abstract boolean getSendIngestMessages();

        abstract void setSendIngestMessages(boolean showInboxMessages);

        public abstract boolean isUpdateable() throws TskCoreException;

        public abstract void addHashes(Content content) throws TskCoreException;

        public abstract void addHashes(Content content, String comment) throws TskCoreException;

        public abstract void addHashes(List<HashEntry> hashes) throws TskCoreException;

        public abstract boolean lookupMD5Quick(Content content) throws TskCoreException;

        public abstract HashHitInfo lookupMD5(Content content) throws TskCoreException;

        abstract boolean isValid() throws TskCoreException;

        public abstract String getIndexPath() throws TskCoreException;

        public abstract boolean hasIndexOnly() throws TskCoreException;

        public abstract void firePropertyChange(String propertyName, Object oldValue, Object newValue);

        public abstract void addPropertyChangeListener(PropertyChangeListener pcl);

        public abstract void removePropertyChangeListener(PropertyChangeListener pcl);

        @Override
        public abstract String toString();

    }

    class SleuthkitHashSet extends HashDb {

        private static final long serialVersionUID = 1L;
        private final int handle;
        private final String hashSetName;
        private boolean searchDuringIngest;
        private boolean sendIngestMessages;
        private final HashDb.KnownFilesType knownFilesType;
        private boolean indexing;
        private final PropertyChangeSupport propertyChangeSupport = new PropertyChangeSupport(this);

        private SleuthkitHashSet(int handle, String hashSetName, boolean useForIngest, boolean sendHitMessages, KnownFilesType knownFilesType) {
            this.handle = handle;
            this.hashSetName = hashSetName;
            this.searchDuringIngest = useForIngest;
            this.sendIngestMessages = sendHitMessages;
            this.knownFilesType = knownFilesType;
            this.indexing = false;
        }

        @Override
        public void addPropertyChangeListener(PropertyChangeListener pcl) {
            propertyChangeSupport.addPropertyChangeListener(pcl);
        }

        @Override
        public void removePropertyChangeListener(PropertyChangeListener pcl) {
            propertyChangeSupport.removePropertyChangeListener(pcl);
        }

        int getHandle() {
            return handle;
        }

        @Override
        public String getHashSetName() {
            return hashSetName;
        }

        @Override
        String getDisplayName() {
            return getHashSetName();
        }

        @Override
        public String getDatabasePath() throws TskCoreException {
            return SleuthkitJNI.getHashDatabasePath(handle);
        }

        public void setIndexing(boolean indexing) {
            this.indexing = indexing;
        }

        @Override
        public String getIndexPath() throws TskCoreException {
            return SleuthkitJNI.getHashDatabaseIndexPath(handle);
        }

        @Override
        public KnownFilesType getKnownFilesType() {
            return knownFilesType;
        }

        @Override
        public boolean getSearchDuringIngest() {
            return searchDuringIngest;
        }

        @Override
        void setSearchDuringIngest(boolean useForIngest) {
            this.searchDuringIngest = useForIngest;
        }

        @Override
        public boolean getSendIngestMessages() {
            return sendIngestMessages;
        }

        @Override
        void setSendIngestMessages(boolean showInboxMessages) {
            this.sendIngestMessages = showInboxMessages;
        }

        @Override
        public boolean isUpdateable() throws TskCoreException {
            return SleuthkitJNI.isUpdateableHashDatabase(this.handle);
        }

        @Override
        public void addHashes(Content content) throws TskCoreException {
            addHashes(content, null);
        }

        @Override
        public void addHashes(Content content, String comment) throws TskCoreException {
            // This only works for AbstractFiles and MD5 hashes at present. 
            assert content instanceof AbstractFile;
            if (content instanceof AbstractFile) {
                AbstractFile file = (AbstractFile) content;
                if (null != file.getMd5Hash()) {
                    SleuthkitJNI.addToHashDatabase(null, file.getMd5Hash(), null, null, comment, handle);
                }
            }
        }

        @Override
        public void addHashes(List<HashEntry> hashes) throws TskCoreException {
            SleuthkitJNI.addToHashDatabase(hashes, handle);
        }

        @Override
        public boolean lookupMD5Quick(Content content) throws TskCoreException {
            boolean result = false;
            assert content instanceof AbstractFile;
            if (content instanceof AbstractFile) {
                AbstractFile file = (AbstractFile) content;
                if (null != file.getMd5Hash()) {
                    result = SleuthkitJNI.lookupInHashDatabase(file.getMd5Hash(), handle);
                }
            }
            return result;
        }

        @Override
        public HashHitInfo lookupMD5(Content content) throws TskCoreException {
            HashHitInfo result = null;
            // This only works for AbstractFiles and MD5 hashes at present. 
            assert content instanceof AbstractFile;
            if (content instanceof AbstractFile) {
                AbstractFile file = (AbstractFile) content;
                if (null != file.getMd5Hash()) {
                    result = SleuthkitJNI.lookupInHashDatabaseVerbose(file.getMd5Hash(), handle);
                }
            }
            return result;
        }

        @Override
        boolean isValid() throws TskCoreException {
            return hasIndex();
        }

        boolean hasIndex() throws TskCoreException {
            return SleuthkitJNI.hashDatabaseHasLookupIndex(handle);
        }

        @Override
        public boolean hasIndexOnly() throws TskCoreException {
            return SleuthkitJNI.hashDatabaseIsIndexOnly(handle);
        }

        boolean canBeReIndexed() throws TskCoreException {
            return SleuthkitJNI.hashDatabaseCanBeReindexed(handle);
        }

        boolean isIndexing() {
            return indexing;
        }

        @Override
        public void firePropertyChange(String propertyName, Object oldValue, Object newValue) {
            this.propertyChangeSupport.firePropertyChange(propertyName, oldValue, newValue);
        }

        private void close() throws TskCoreException {
            SleuthkitJNI.closeHashDatabase(handle);
        }

        @Override
        public String toString() {
            return getHashSetName();
        }

        @Override
        public int hashCode() {
            int code = 23;
            code = 47 * code + Integer.hashCode(handle);
            code = 47 * code + Objects.hashCode(this.hashSetName);
            code = 47 * code + Objects.hashCode(this.propertyChangeSupport);
            code = 47 * code + Objects.hashCode(this.knownFilesType);
            return code;
        }

        @Override
        public boolean equals(Object obj) {
            if (obj == null) {
                return false;
            }
            if (getClass() != obj.getClass()) {
                return false;
            }
            final SleuthkitHashSet other = (SleuthkitHashSet) obj;
            if (!Objects.equals(this.hashSetName, other.hashSetName)) {
                return false;
            }
            if (this.knownFilesType != other.knownFilesType) {
                return false;
            }
            return true;
        }
    }

    class CentralRepoHashSet extends HashDb {

        private static final long serialVersionUID = 1L;
        private final String hashSetName;
        private boolean searchDuringIngest;
        private boolean sendIngestMessages;
        private final HashDb.KnownFilesType knownFilesType;
        private final int referenceSetID;
        private final String version;
        private String orgName;
        private final boolean readOnly;
        private final PropertyChangeSupport propertyChangeSupport = new PropertyChangeSupport(this);

        @Messages({"HashDbManager.CentralRepoHashDb.orgError=Error loading organization"})
        private CentralRepoHashSet(String hashSetName, String version, int referenceSetID,
                boolean useForIngest, boolean sendHitMessages, HashDb.KnownFilesType knownFilesType,
                boolean readOnly)
                throws TskCoreException {
            this.hashSetName = hashSetName;
            this.version = version;
            this.referenceSetID = referenceSetID;
            this.searchDuringIngest = useForIngest;
            this.sendIngestMessages = sendHitMessages;
            this.knownFilesType = knownFilesType;
            this.readOnly = readOnly;

            try {
                orgName = CentralRepository.getInstance().getReferenceSetOrganization(referenceSetID).getName();
            } catch (CentralRepoException ex) {
                Logger.getLogger(SleuthkitHashSet.class.getName()).log(Level.SEVERE, "Error looking up central repository organization for reference set " + referenceSetID, ex); //NON-NLS
                orgName = Bundle.HashDbManager_CentralRepoHashDb_orgError();
            }
        }

        @Override
        public void addPropertyChangeListener(PropertyChangeListener pcl) {
            propertyChangeSupport.addPropertyChangeListener(pcl);
        }

        @Override
        public void removePropertyChangeListener(PropertyChangeListener pcl) {
            propertyChangeSupport.removePropertyChangeListener(pcl);
        }

        @Override
        public boolean hasIndexOnly() throws TskCoreException {
            return true;
        }

        @Override
        public String getHashSetName() {
            return hashSetName;
        }

        @Override
        public String getDisplayName() {
            if (!getVersion().isEmpty()) {
                return getHashSetName() + " " + getVersion() + " (remote)";
            } else {
                return getHashSetName() + " (remote)";
            }
        }

        String getVersion() {
            return version;
        }

        String getOrgName() {
            return orgName;
        }

        int getReferenceSetID() {
            return referenceSetID;
        }

        @Override
        public String getDatabasePath() throws TskCoreException {
            return "";
        }

        @Override
        public String getIndexPath() throws TskCoreException {
            return "";
        }

        @Override
        public HashDb.KnownFilesType getKnownFilesType() {
            return knownFilesType;
        }

        @Override
        public boolean getSearchDuringIngest() {
            return searchDuringIngest;
        }

        @Override
        void setSearchDuringIngest(boolean useForIngest) {
            this.searchDuringIngest = useForIngest;
        }

        @Override
        public boolean getSendIngestMessages() {
            return sendIngestMessages;
        }

        @Override
        void setSendIngestMessages(boolean showInboxMessages) {
            this.sendIngestMessages = showInboxMessages;
        }

        @Override
        public boolean isUpdateable() throws TskCoreException {
            return (!readOnly);
        }

        @Override
        public void addHashes(Content content) throws TskCoreException {
            addHashes(content, null);
        }

        @Override
        public void addHashes(Content content, String comment) throws TskCoreException {
            // This only works for AbstractFiles and MD5 hashes at present. 
            assert content instanceof AbstractFile;
            if (content instanceof AbstractFile) {
                AbstractFile file = (AbstractFile) content;
                if (null != file.getMd5Hash()) {
                    TskData.FileKnown type = knownFilesType.getFileKnown();

                    try {
                        CentralRepoFileInstance fileInstance = new CentralRepoFileInstance(referenceSetID, file.getMd5Hash(),
                                type, comment);
                        CentralRepository.getInstance().addReferenceInstance(fileInstance, CentralRepository.getInstance().getCorrelationTypeById(CorrelationAttributeInstance.FILES_TYPE_ID));
                    } catch (CentralRepoException | CorrelationAttributeNormalizationException ex) {
                        throw new TskCoreException("Error adding hashes to " + getDisplayName(), ex);   //NON-NLS
                    }
                }
            }
        }

        @Override
        public void addHashes(List<HashEntry> hashes) throws TskCoreException {
            Set<CentralRepoFileInstance> globalFileInstances = new HashSet<>();
            for (HashEntry hashEntry : hashes) {
                TskData.FileKnown type = knownFilesType.getFileKnown();

                try {
                    globalFileInstances.add(new CentralRepoFileInstance(referenceSetID, hashEntry.getMd5Hash(), type, hashEntry.getComment()));
                } catch (CentralRepoException | CorrelationAttributeNormalizationException ex) {
                    throw new TskCoreException("Error adding hashes to " + getDisplayName(), ex);
                }
            }

            try {
                CentralRepository.getInstance().bulkInsertReferenceTypeEntries(globalFileInstances,
                        CentralRepository.getInstance().getCorrelationTypeById(CorrelationAttributeInstance.FILES_TYPE_ID));
            } catch (CentralRepoException ex) {
                throw new TskCoreException("Error adding hashes to " + getDisplayName(), ex);
            }
        }

        @Override
        public boolean lookupMD5Quick(Content content) throws TskCoreException {
            // This only works for AbstractFiles and MD5 hashes 
            assert content instanceof AbstractFile;
            if (content instanceof AbstractFile) {
                AbstractFile file = (AbstractFile) content;
                if (null != file.getMd5Hash()) {
                    try {
                        return CentralRepository.getInstance().isFileHashInReferenceSet(file.getMd5Hash(), this.referenceSetID);
                    } catch (CentralRepoException | CorrelationAttributeNormalizationException ex) {
                        Logger.getLogger(SleuthkitHashSet.class.getName()).log(Level.SEVERE, "Error performing central reposiotry hash lookup for hash "
                                + file.getMd5Hash() + " in reference set " + referenceSetID, ex); //NON-NLS
                        throw new TskCoreException("Error performing central reposiotry hash lookup", ex);
                    }
                }
            }
            return false;
        }

        @Override
        public HashHitInfo lookupMD5(Content content) throws TskCoreException {
            HashHitInfo result = null;
            // This only works for AbstractFiles and MD5 hashes 
            assert content instanceof AbstractFile;
            if (content instanceof AbstractFile) {
                AbstractFile file = (AbstractFile) content;
                if (null != file.getMd5Hash()) {
                    try {
                        return CentralRepository.getInstance().lookupHash(file.getMd5Hash(), referenceSetID);
                    } catch (CentralRepoException | CorrelationAttributeNormalizationException ex) {
                        Logger.getLogger(SleuthkitHashSet.class.getName()).log(Level.SEVERE, "Error performing central reposiotry hash lookup for hash "
                                + file.getMd5Hash() + " in reference set " + referenceSetID, ex); //NON-NLS
                        throw new TskCoreException("Error performing central reposiotry hash lookup", ex);
                    }
                }
            }
            return result;
        }

        @Override
        boolean isValid() {
            if (!CentralRepository.isEnabled()) {
                return false;
            }
            try {
                return CentralRepository.getInstance().referenceSetIsValid(this.referenceSetID, this.hashSetName, this.version);
            } catch (CentralRepoException ex) {
                Logger.getLogger(CentralRepoHashSet.class.getName()).log(Level.SEVERE, "Error validating hash set " + hashSetName, ex); //NON-NLS
                return false;
            }
        }

        @Override
        public void firePropertyChange(String propertyName, Object oldValue, Object newValue) {
            this.propertyChangeSupport.firePropertyChange(propertyName, oldValue, newValue);
        }

        @Override
        public String toString() {
            return getDisplayName();
        }

        @Override
        public int hashCode() {
            int code = 23;
            code = 47 * code + Objects.hashCode(this.hashSetName);
            code = 47 * code + Objects.hashCode(this.version);
            code = 47 * code + Integer.hashCode(this.referenceSetID);
            code = 47 * code + Objects.hashCode(this.knownFilesType);
            return code;
        }

        @Override
        public boolean equals(Object obj) {
            if (obj == null) {
                return false;
            }
            if (getClass() != obj.getClass()) {
                return false;
            }
            final CentralRepoHashSet other = (CentralRepoHashSet) obj;
            if (!Objects.equals(this.hashSetName, other.hashSetName)) {
                return false;
            }
            if (!Objects.equals(this.version, other.version)) {
                return false;
            }
            if (this.knownFilesType != other.knownFilesType) {
                return false;
            }
            return true;
        }
    }

    private class HashDbIndexer extends SwingWorker<Object, Void> {

        private ProgressHandle progress = null;
        private SleuthkitHashSet hashDb = null;

        HashDbIndexer(SleuthkitHashSet hashDb) {
            this.hashDb = hashDb;
        }

        @Override
        protected Object doInBackground() {
            hashDb.setIndexing(true);
            progress = ProgressHandle.createHandle(
                    NbBundle.getMessage(this.getClass(), "HashDbManager.progress.indexingHashSet", hashDb.getHashSetName()));
            progress.start();
            progress.switchToIndeterminate();
            try {
                SleuthkitJNI.createLookupIndexForHashDatabase(hashDb.getHandle());
            } catch (TskCoreException ex) {
                Logger.getLogger(HashDbIndexer.class.getName()).log(Level.SEVERE, "Error indexing hash set " + hashDb.getHashSetName(), ex); //NON-NLS
                JOptionPane.showMessageDialog(WindowManager.getDefault().getMainWindow(),
                        NbBundle.getMessage(this.getClass(),
                                "HashDbManager.dlgMsg.errorIndexingHashSet",
                                hashDb.getHashSetName()),
                        NbBundle.getMessage(this.getClass(), "HashDbManager.hashDbIndexingErr"),
                        JOptionPane.ERROR_MESSAGE);
            }
            return null;
        }

        @Override
        protected void done() {
            hashDb.setIndexing(false);
            progress.finish();

            // see if we got any errors
            try {
                get();
            } catch (InterruptedException | ExecutionException ex) {
                logger.log(Level.SEVERE, "Error creating index", ex); //NON-NLS
                MessageNotifyUtil.Notify.show(
                        NbBundle.getMessage(this.getClass(), "HashDbManager.errCreatingIndex.title"),
                        NbBundle.getMessage(this.getClass(), "HashDbManager.errCreatingIndex.msg", ex.getMessage()),
                        MessageNotifyUtil.MessageType.ERROR);
            } // catch and ignore if we were cancelled
            catch (java.util.concurrent.CancellationException ex) {
            }

            try {
                hashDb.firePropertyChange(SleuthkitHashSet.Event.INDEXING_DONE.toString(), null, hashDb);
                hashDb.firePropertyChange(HashDbManager.SetEvt.DB_INDEXED.toString(), null, hashDb.getHashSetName());
            } catch (Exception e) {
                logger.log(Level.SEVERE, "HashDbManager listener threw exception", e); //NON-NLS
                MessageNotifyUtil.Notify.show(
                        NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErr"),
                        NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErrorListeningToUpdatesMsg"),
                        MessageNotifyUtil.MessageType.ERROR);
            }
        }
    }
}