HashDbManager.java

Go to the documentation of this file.

/*
 * Autopsy Forensic Browser
 * 
 * Copyright 2011-2018 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.modules.hashdatabase;

import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.beans.PropertyChangeSupport;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.logging.Level;
import javax.swing.JFileChooser;
import javax.swing.JOptionPane;
import javax.swing.SwingWorker;
import javax.swing.filechooser.FileNameExtensionFilter;
import org.apache.commons.io.FilenameUtils;
import org.netbeans.api.progress.ProgressHandle;
import org.openide.util.NbBundle;
import org.openide.util.NbBundle.Messages;
import org.openide.windows.WindowManager;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamGlobalFileInstance;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamGlobalSet;
import org.sleuthkit.autopsy.core.RuntimeProperties;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.ingest.IngestManager;
import org.sleuthkit.autopsy.modules.hashdatabase.HashLookupSettings.HashDbInfo;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.HashEntry;
import org.sleuthkit.datamodel.HashHitInfo;
import org.sleuthkit.datamodel.SleuthkitJNI;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;

public class HashDbManager implements PropertyChangeListener {

    private static final String HASH_DATABASE_FILE_EXTENSON = "kdb"; //NON-NLS
    private static HashDbManager instance = null;
    private List<HashDb> hashSets = new ArrayList<>();
    private Set<String> hashSetNames = new HashSet<>();
    private Set<String> hashSetPaths = new HashSet<>();
    PropertyChangeSupport changeSupport = new PropertyChangeSupport(HashDbManager.class);
    private static final Logger logger = Logger.getLogger(HashDbManager.class.getName());
    private boolean allDatabasesLoadedCorrectly = false;

    public enum SetEvt {

        DB_ADDED, DB_DELETED, DB_INDEXED
    };

    public static synchronized HashDbManager getInstance() {
        if (instance == null) {
            instance = new HashDbManager();
        }
        return instance;
    }

    public synchronized void addPropertyChangeListener(PropertyChangeListener listener) {
        changeSupport.addPropertyChangeListener(listener);
    }

    public synchronized void removePropertyChangeListener(PropertyChangeListener listener) {
        changeSupport.removePropertyChangeListener(listener);
    }
    
    synchronized boolean verifyAllDatabasesLoadedCorrectly(){
        return allDatabasesLoadedCorrectly;
    }

    private HashDbManager() {
        loadHashsetsConfiguration();
    }

    static String getHashDatabaseFileExtension() {
        return HASH_DATABASE_FILE_EXTENSON;
    }

    public class HashDbManagerException extends Exception {

        private static final long serialVersionUID = 1L;

        private HashDbManagerException(String message) {
            super(message);
        }

        private HashDbManagerException(String message, Throwable exception) {
            super(message, exception);
        }
    }

    public synchronized HashDb addExistingHashDatabase(String hashSetName, String path, boolean searchDuringIngest, boolean sendIngestMessages, HashDb.KnownFilesType knownFilesType) throws HashDbManagerException {
        HashDb hashDb = null;
        hashDb = this.addExistingHashDatabaseNoSave(hashSetName, path, searchDuringIngest, sendIngestMessages, knownFilesType);
        this.save();
        return hashDb;
    }

    synchronized HashDb addExistingHashDatabaseNoSave(String hashSetName, String path, boolean searchDuringIngest, boolean sendIngestMessages, HashDb.KnownFilesType knownFilesType) throws HashDbManagerException {
        HashDb hashDb = null;
        try {
            if (!new File(path).exists()) {
                throw new HashDbManagerException(NbBundle.getMessage(HashDbManager.class, "HashDbManager.hashDbDoesNotExistExceptionMsg", path));
            }

            if (hashSetPaths.contains(path)) {
                throw new HashDbManagerException(NbBundle.getMessage(HashDbManager.class, "HashDbManager.hashDbAlreadyAddedExceptionMsg", path));
            }

            if (hashSetNames.contains(hashSetName)) {
                throw new HashDbManagerException(NbBundle.getMessage(HashDbManager.class, "HashDbManager.duplicateHashSetNameExceptionMsg", hashSetName));
            }

            hashDb = addHashDatabase(SleuthkitJNI.openHashDatabase(path), hashSetName, searchDuringIngest, sendIngestMessages, knownFilesType);
        } catch (TskCoreException ex) {
            throw new HashDbManagerException(ex.getMessage());
        }
        return hashDb;
    }

    public synchronized HashDb addNewHashDatabase(String hashSetName, String path, boolean searchDuringIngest, boolean sendIngestMessages,
            HashDb.KnownFilesType knownFilesType) throws HashDbManagerException {

        HashDb hashDb = null;
        hashDb = this.addNewHashDatabaseNoSave(hashSetName, path, searchDuringIngest, sendIngestMessages, knownFilesType);

        this.save();

        return hashDb;
    }

    public synchronized HashDb addNewHashDatabaseNoSave(String hashSetName, String path, boolean searchDuringIngest, boolean sendIngestMessages,
            HashDb.KnownFilesType knownFilesType) throws HashDbManagerException {
        HashDb hashDb = null;
        try {
            File file = new File(path);
            if (file.exists()) {
                throw new HashDbManagerException(NbBundle.getMessage(HashDbManager.class, "HashDbManager.hashDbFileExistsExceptionMsg", path));
            }
            if (!FilenameUtils.getExtension(file.getName()).equalsIgnoreCase(HASH_DATABASE_FILE_EXTENSON)) {
                throw new HashDbManagerException(NbBundle.getMessage(HashDbManager.class, "HashDbManager.illegalHashDbFileNameExtensionMsg",
                        getHashDatabaseFileExtension()));
            }

            if (hashSetPaths.contains(path)) {
                throw new HashDbManagerException(NbBundle.getMessage(HashDbManager.class, "HashDbManager.hashDbAlreadyAddedExceptionMsg", path));
            }

            if (hashSetNames.contains(hashSetName)) {
                throw new HashDbManagerException(NbBundle.getMessage(HashDbManager.class, "HashDbManager.duplicateHashSetNameExceptionMsg", hashSetName));
            }

            hashDb = addHashDatabase(SleuthkitJNI.createHashDatabase(path), hashSetName, searchDuringIngest, sendIngestMessages, knownFilesType);
        } catch (TskCoreException ex) {
            throw new HashDbManagerException(ex.getMessage());
        }
        return hashDb;
    }
    
    private SleuthkitHashSet addHashDatabase(int handle, String hashSetName, boolean searchDuringIngest, boolean sendIngestMessages, HashDb.KnownFilesType knownFilesType) throws TskCoreException {
        // Wrap an object around the handle.
        SleuthkitHashSet hashDb = new SleuthkitHashSet(handle, hashSetName, searchDuringIngest, sendIngestMessages, knownFilesType);

        // Get the indentity data before updating the collections since the 
        // accessor methods may throw. 
        String databasePath = hashDb.getDatabasePath();
        String indexPath = hashDb.getIndexPath();

        // Update the collections used to ensure that hash set names are unique 
        // and the same database is not added to the configuration more than once.
        hashSetNames.add(hashDb.getHashSetName());
        if (!databasePath.equals("None")) { //NON-NLS
            hashSetPaths.add(databasePath);
        }
        if (!indexPath.equals("None")) { //NON-NLS
            hashSetPaths.add(indexPath);
        }

        // Add the hash database to the collection
        hashSets.add(hashDb);

        // Let any external listeners know that there's a new set   
        try {
            changeSupport.firePropertyChange(SetEvt.DB_ADDED.toString(), null, hashSetName);
        } catch (Exception e) {
            logger.log(Level.SEVERE, "HashDbManager listener threw exception", e); //NON-NLS
            MessageNotifyUtil.Notify.show(
                    NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErr"),
                    NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErrorListeningToUpdatesMsg"),
                    MessageNotifyUtil.MessageType.ERROR);
        }
        return hashDb;
    }
    
    CentralRepoHashSet addExistingCentralRepoHashSet(String hashSetName, String version, int referenceSetID, 
            boolean searchDuringIngest, boolean sendIngestMessages, HashDb.KnownFilesType knownFilesType, 
            boolean readOnly) throws TskCoreException{
        
        if(! EamDb.isEnabled()){
            throw new TskCoreException("Could not load central repository hash set " + hashSetName + " - central repository is not enabled");
        }
        
        CentralRepoHashSet db = new CentralRepoHashSet(hashSetName, version, referenceSetID, searchDuringIngest,
            sendIngestMessages, knownFilesType, readOnly);
        
        if(! db.isValid()){
            throw new TskCoreException("Error finding hash set " + hashSetName + " in central repository");
        }
        
        // Add the hash database to the collection
        hashSets.add(db);

        // Let any external listeners know that there's a new set   
        try {
            changeSupport.firePropertyChange(SetEvt.DB_ADDED.toString(), null, hashSetName);
        } catch (Exception e) {
            logger.log(Level.SEVERE, "HashDbManager listener threw exception", e); //NON-NLS
            MessageNotifyUtil.Notify.show(
                    NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErr"),
                    NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErrorListeningToUpdatesMsg"),
                    MessageNotifyUtil.MessageType.ERROR);
        }
        return db;        
        
    }

    synchronized void indexHashDatabase(SleuthkitHashSet hashDb) {
        hashDb.addPropertyChangeListener(this);
        HashDbIndexer creator = new HashDbIndexer(hashDb);
        creator.execute();
    }

    @Override
    public void propertyChange(PropertyChangeEvent event) {
        if (event.getPropertyName().equals(SleuthkitHashSet.Event.INDEXING_DONE.name())) {
            SleuthkitHashSet hashDb = (SleuthkitHashSet) event.getNewValue();
            if (null != hashDb) {
                try {
                    String indexPath = hashDb.getIndexPath();
                    if (!indexPath.equals("None")) { //NON-NLS
                        hashSetPaths.add(indexPath);
                    }
                } catch (TskCoreException ex) {
                    Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error getting index path of " + hashDb.getHashSetName() + " hash set after indexing", ex); //NON-NLS
                }
            }
        }
    }

    public synchronized void removeHashDatabase(HashDb hashDb) throws HashDbManagerException {
        this.removeHashDatabaseNoSave(hashDb);
        this.save();
    }
    
    public synchronized void removeHashDatabaseNoSave(HashDb hashDb) throws HashDbManagerException {
        // Don't remove a database if ingest is running
        boolean ingestIsRunning = IngestManager.getInstance().isIngestRunning();
        if (ingestIsRunning) {
            throw new HashDbManagerException(NbBundle.getMessage(this.getClass(), "HashDbManager.ingestRunningExceptionMsg"));
        }
        // Remove the database from whichever hash set list it occupies,
        // and remove its hash set name from the hash set used to ensure unique
        // hash set names are used, before undertaking These operations will succeed and constitute
        // a mostly effective removal, even if the subsequent operations fail.
        String hashSetName = hashDb.getHashSetName();
        hashSetNames.remove(hashSetName);
        hashSets.remove(hashDb);

        // Now undertake the operations that could throw.
        
        // Indexing is only relevanet for sleuthkit hashsets
        if(hashDb instanceof SleuthkitHashSet){
            SleuthkitHashSet hashDatabase = (SleuthkitHashSet)hashDb;
            try {
                if(hashDatabase.hasIndex()){
                    hashSetPaths.remove(hashDatabase.getIndexPath());
                }
            } catch (TskCoreException ex) {
                Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error getting index path of " + hashDatabase.getHashSetName() + " hash set when removing the hash set", ex); //NON-NLS
            }        

            try {
                if (!hashDatabase.hasIndexOnly()) {
                    hashSetPaths.remove(hashDatabase.getDatabasePath());
                }
            } catch (TskCoreException ex) {
                Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error getting hash set path of " + hashDatabase.getHashSetName() + " hash set when removing the hash set", ex); //NON-NLS
            }
        
            try {
                hashDatabase.close();
            } catch (TskCoreException ex) {
                Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error closing " + hashDb.getHashSetName() + " hash set when removing the hash set", ex); //NON-NLS
            }
        }

        // Let any external listeners know that a set has been deleted
        try {
            changeSupport.firePropertyChange(SetEvt.DB_DELETED.toString(), null, hashSetName);
        } catch (Exception e) {
            logger.log(Level.SEVERE, "HashDbManager listener threw exception", e); //NON-NLS
            MessageNotifyUtil.Notify.show(
                    NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErr"),
                    NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErrorListeningToUpdatesMsg"),
                    MessageNotifyUtil.MessageType.ERROR);
        }
    }

    void save() throws HashDbManagerException {
        try {
            if (!HashLookupSettings.writeSettings(new HashLookupSettings(HashLookupSettings.convertHashSetList(this.hashSets)))) {
                throw new HashDbManagerException(NbBundle.getMessage(this.getClass(), "HashDbManager.saveErrorExceptionMsg"));
            }
        } catch (HashLookupSettings.HashLookupSettingsException ex) {
            throw new HashDbManagerException(NbBundle.getMessage(this.getClass(), "HashDbManager.saveErrorExceptionMsg"));
        }
    }
    
    public synchronized List<HashDb> getAllHashSets() {
        try{
            updateHashSetsFromCentralRepository();
        } catch (TskCoreException ex){
            Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error loading central repository hash sets", ex); //NON-NLS
        }
        
        List<HashDb> hashDbs = new ArrayList<>();
        hashDbs.addAll(this.hashSets);
        return hashDbs;
    }

    public synchronized List<HashDb> getKnownFileHashSets() {
        List<HashDb> hashDbs = new ArrayList<>();
        try{
            updateHashSetsFromCentralRepository();
        } catch (TskCoreException ex){
            Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error loading central repository hash sets", ex); //NON-NLS
        }
        this.hashSets.stream().filter((db) -> (db.getKnownFilesType() == HashDb.KnownFilesType.KNOWN)).forEach((db) -> {
            hashDbs.add(db);
        });
        return hashDbs;
    }

    public synchronized List<HashDb> getKnownBadFileHashSets() {
        List<HashDb> hashDbs = new ArrayList<>();
        try{
            updateHashSetsFromCentralRepository();
        } catch (TskCoreException ex){
            Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error loading central repository hash sets", ex); //NON-NLS
        }
        this.hashSets.stream().filter((db) -> (db.getKnownFilesType() == HashDb.KnownFilesType.KNOWN_BAD)).forEach((db) -> {
            hashDbs.add(db);
        });
        return hashDbs;
    }

    public synchronized List<HashDb> getUpdateableHashSets() {
        return getUpdateableHashSets(this.hashSets);
    }

    private List<HashDb> getUpdateableHashSets(List<HashDb> hashDbs) {
        ArrayList<HashDb> updateableDbs = new ArrayList<>();
        try{
            updateHashSetsFromCentralRepository();
        } catch (TskCoreException ex){
            Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error loading central repository hash sets", ex); //NON-NLS
        }
        for (HashDb db : hashDbs) {
            try {
                if (db.isUpdateable()) {
                    updateableDbs.add(db);
                }
            } catch (TskCoreException ex) {
                Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error checking updateable status of " + db.getHashSetName() + " hash set", ex); //NON-NLS
            }
        }
        return updateableDbs;
    }
    
    private List<HashDbInfo> getCentralRepoHashSetsFromDatabase(){
        List<HashDbInfo> crHashSets = new ArrayList<>();
        if(EamDb.isEnabled()){
            try{
                List<EamGlobalSet> crSets = EamDb.getInstance().getAllReferenceSets(EamDb.getInstance().getCorrelationTypeById(CorrelationAttributeInstance.FILES_TYPE_ID));
                for(EamGlobalSet globalSet:crSets){
                    
                    // Defaults for fields not stored in the central repository:
                    //   searchDuringIngest: false
                    //   sendIngestMessages: true if the hash set is notable
                    boolean sendIngestMessages = convertFileKnown(globalSet.getFileKnownStatus()).equals(HashDb.KnownFilesType.KNOWN_BAD);
                    crHashSets.add(new HashDbInfo(globalSet.getSetName(), globalSet.getVersion(),
                        globalSet.getGlobalSetID(), convertFileKnown(globalSet.getFileKnownStatus()), globalSet.isReadOnly(), false, sendIngestMessages));
               }
            } catch (EamDbException ex){
                Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error loading central repository hash sets", ex); //NON-NLS
            }
        }
        return crHashSets;
    }
    
    private static HashDb.KnownFilesType convertFileKnown(TskData.FileKnown fileKnown){
        if(fileKnown.equals(TskData.FileKnown.BAD)){
            return HashDb.KnownFilesType.KNOWN_BAD;
        }
        return HashDb.KnownFilesType.KNOWN;
    }

    public synchronized void loadLastSavedConfiguration() {
        closeHashDatabases(this.hashSets);
        hashSetNames.clear();
        hashSetPaths.clear();

        loadHashsetsConfiguration();
    }

    private void closeHashDatabases(List<HashDb> hashDatabases) {
        for (HashDb database : hashDatabases) {
            if(database instanceof SleuthkitHashSet){
                try {
                    ((SleuthkitHashSet)database).close();
                } catch (TskCoreException ex) {
                    Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error closing " + database.getHashSetName() + " hash set", ex); //NON-NLS
                }
            }
        }
        hashDatabases.clear();
    }

    private void loadHashsetsConfiguration() {
        try {
            HashLookupSettings settings = HashLookupSettings.readSettings();
            this.configureSettings(settings);
        } catch (HashLookupSettings.HashLookupSettingsException ex) {
            Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Could not read Hash lookup settings from disk.", ex);
        }
    }

    @Messages({"# {0} - hash set name", "HashDbManager.noDbPath.message=Couldn't get valid hash set path for: {0}",
            "HashDbManager.centralRepoLoadError.message=Error loading central repository hash sets"})
    private void configureSettings(HashLookupSettings settings) {
        allDatabasesLoadedCorrectly = true;
        List<HashDbInfo> hashDbInfoList = settings.getHashDbInfo();
        for (HashDbInfo hashDbInfo : hashDbInfoList) {
            try {
                if(hashDbInfo.isFileDatabaseType()){
                    String dbPath = this.getValidFilePath(hashDbInfo.getHashSetName(), hashDbInfo.getPath());
                    if (dbPath != null) {
                        addHashDatabase(SleuthkitJNI.openHashDatabase(dbPath), hashDbInfo.getHashSetName(), hashDbInfo.getSearchDuringIngest(), hashDbInfo.getSendIngestMessages(), hashDbInfo.getKnownFilesType());
                    } else {
                        logger.log(Level.WARNING, Bundle.HashDbManager_noDbPath_message(hashDbInfo.getHashSetName()));
                        allDatabasesLoadedCorrectly = false;
                    }
                } else {
                    if(EamDb.isEnabled()){
                        addExistingCentralRepoHashSet(hashDbInfo.getHashSetName(), hashDbInfo.getVersion(), 
                                hashDbInfo.getReferenceSetID(), 
                                hashDbInfo.getSearchDuringIngest(), hashDbInfo.getSendIngestMessages(), 
                                hashDbInfo.getKnownFilesType(), hashDbInfo.isReadOnly());
                    }
                }
            } catch (TskCoreException ex) {
                Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error opening hash set", ex); //NON-NLS
                JOptionPane.showMessageDialog(WindowManager.getDefault().getMainWindow(),
                        NbBundle.getMessage(this.getClass(),
                                "HashDbManager.unableToOpenHashDbMsg", hashDbInfo.getHashSetName()),
                        NbBundle.getMessage(this.getClass(), "HashDbManager.openHashDbErr"),
                        JOptionPane.ERROR_MESSAGE);
                allDatabasesLoadedCorrectly = false;
            }
        }
        
        if(EamDb.isEnabled()){
            try{
                updateHashSetsFromCentralRepository();
            } catch (TskCoreException ex){
                Logger.getLogger(HashDbManager.class.getName()).log(Level.SEVERE, "Error opening hash set", ex); //NON-NLS
                
                JOptionPane.showMessageDialog(WindowManager.getDefault().getMainWindow(),
                        Bundle.HashDbManager_centralRepoLoadError_message(),
                        NbBundle.getMessage(this.getClass(), "HashDbManager.openHashDbErr"),
                        JOptionPane.ERROR_MESSAGE);
                allDatabasesLoadedCorrectly = false;
            }
        }
        
        /* NOTE: When RuntimeProperties.coreComponentsAreActive() is "false", 
        I don't think we should overwrite hash db settings file because we 
        were unable to load a database. The user should have to fix the issue or 
        remove the database from settings. Overwiting the settings effectively removes 
        the database from HashLookupSettings and the user may not know about this 
        because the dialogs are not being displayed. The next time user starts Autopsy, HashDB 
        will load without errors and the user may think that the problem was solved.*/
        if (!allDatabasesLoadedCorrectly && RuntimeProperties.runningWithGUI()) {
            try {
                HashLookupSettings.writeSettings(new HashLookupSettings(HashLookupSettings.convertHashSetList(this.hashSets)));
                allDatabasesLoadedCorrectly = true;
            } catch (HashLookupSettings.HashLookupSettingsException ex) {
                allDatabasesLoadedCorrectly = false;
                logger.log(Level.SEVERE, "Could not overwrite hash set settings.", ex);
            }
        }
    }
    
    private void updateHashSetsFromCentralRepository() throws TskCoreException {
        if(EamDb.isEnabled()){
            List<HashDbInfo> crHashDbInfoList = getCentralRepoHashSetsFromDatabase();
            for(HashDbInfo hashDbInfo : crHashDbInfoList) {
                if(hashDbInfoIsNew(hashDbInfo)){
                    addExistingCentralRepoHashSet(hashDbInfo.getHashSetName(), hashDbInfo.getVersion(), 
                                hashDbInfo.getReferenceSetID(), 
                                hashDbInfo.getSearchDuringIngest(), hashDbInfo.getSendIngestMessages(), hashDbInfo.getKnownFilesType(),
                                hashDbInfo.isReadOnly());   
                }
            }
        }
    }
        
    private boolean hashDbInfoIsNew(HashDbInfo dbInfo){
        for(HashDb db:this.hashSets){
            if(dbInfo.matches(db)){
                return false;
            }
        }
        return true;
    }

    private String getValidFilePath(String hashSetName, String configuredPath) {        
        // Check the configured path.
        File database = new File(configuredPath);
        if (database.exists()) {
            return configuredPath;
        }

        // Give the user an opportunity to find the desired file.
        String newPath = null;
        if (RuntimeProperties.runningWithGUI() && 
                JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(),
                NbBundle.getMessage(this.getClass(), "HashDbManager.dlgMsg.dbNotFoundAtLoc",
                        hashSetName, configuredPath),
                NbBundle.getMessage(this.getClass(), "HashDbManager.dlgTitle.MissingDb"),
                JOptionPane.YES_NO_OPTION) == JOptionPane.YES_OPTION) {
            newPath = searchForFile();
            if (null != newPath && !newPath.isEmpty()) {
                database = new File(newPath);
                if (!database.exists()) {
                    newPath = null;
                }
            }
        }
        return newPath;
    }

    private String searchForFile() {
        String filePath = null;
        JFileChooser fc = new JFileChooser();
        fc.setDragEnabled(false);
        fc.setFileSelectionMode(JFileChooser.FILES_ONLY);
        String[] EXTENSION = new String[]{"txt", "idx", "hash", "Hash", "kdb"}; //NON-NLS
        FileNameExtensionFilter filter = new FileNameExtensionFilter(
                NbBundle.getMessage(this.getClass(), "HashDbManager.fileNameExtensionFilter.title"), EXTENSION);
        fc.setFileFilter(filter);
        fc.setMultiSelectionEnabled(false);
        if (fc.showOpenDialog(null) == JFileChooser.APPROVE_OPTION) {
            File f = fc.getSelectedFile();
            try {
                filePath = f.getCanonicalPath();
            } catch (IOException ex) {
                Logger.getLogger(HashDbManager.class.getName()).log(Level.WARNING, "Couldn't get selected file path", ex); //NON-NLS
            }
        }
        return filePath;
    }
    
    public static abstract class HashDb {
        
        public enum KnownFilesType {

            KNOWN(NbBundle.getMessage(HashDbManager.class, "HashDbManager.known.text")),
            KNOWN_BAD(NbBundle.getMessage(HashDbManager.class, "HashDbManager.knownBad.text"));
            private final String displayName;

            private KnownFilesType(String displayName) {
                this.displayName = displayName;
            }

            public String getDisplayName() {
                return this.displayName;
            }
        }

        public enum Event {

            INDEXING_DONE
        }
        
        public abstract String getHashSetName();
        
        abstract String getDisplayName();

        public abstract String getDatabasePath() throws TskCoreException;

        public abstract HashDb.KnownFilesType getKnownFilesType();

        public abstract boolean getSearchDuringIngest();
                
        abstract void setSearchDuringIngest(boolean useForIngest);

        public abstract boolean getSendIngestMessages();

        abstract void setSendIngestMessages(boolean showInboxMessages);

        public abstract boolean isUpdateable() throws TskCoreException;

        public abstract void addHashes(Content content) throws TskCoreException;

        public abstract void addHashes(Content content, String comment) throws TskCoreException;

        public abstract void addHashes(List<HashEntry> hashes) throws TskCoreException;

        public abstract boolean lookupMD5Quick(Content content) throws TskCoreException;

        public abstract HashHitInfo lookupMD5(Content content) throws TskCoreException;
        
        abstract boolean isValid() throws TskCoreException;
        
        public abstract String getIndexPath() throws TskCoreException;
        
        public abstract boolean hasIndexOnly() throws TskCoreException;
        
        public abstract void firePropertyChange(String propertyName, Object oldValue, Object newValue);
        
        public abstract void addPropertyChangeListener(PropertyChangeListener pcl);
        
        public abstract void removePropertyChangeListener(PropertyChangeListener pcl);
        
        @Override
        public abstract String toString();
        
    }

    class SleuthkitHashSet extends HashDb{
        
        private static final long serialVersionUID = 1L;
        private final int handle;
        private final String hashSetName;
        private boolean searchDuringIngest;
        private boolean sendIngestMessages;
        private final HashDb.KnownFilesType knownFilesType;  
        private boolean indexing;
        private final PropertyChangeSupport propertyChangeSupport = new PropertyChangeSupport(this);

        private SleuthkitHashSet(int handle, String hashSetName, boolean useForIngest, boolean sendHitMessages, KnownFilesType knownFilesType) {
            this.handle = handle;
            this.hashSetName = hashSetName;
            this.searchDuringIngest = useForIngest;
            this.sendIngestMessages = sendHitMessages;
            this.knownFilesType = knownFilesType;
            this.indexing = false;
        }

        @Override
        public void addPropertyChangeListener(PropertyChangeListener pcl) {
            propertyChangeSupport.addPropertyChangeListener(pcl);
        }

        @Override
        public void removePropertyChangeListener(PropertyChangeListener pcl) {
            propertyChangeSupport.removePropertyChangeListener(pcl);
        }
        
        int getHandle(){
            return handle;
        }

        @Override
        public String getHashSetName() {
            return hashSetName;
        }
        
        @Override
        String getDisplayName(){
            return getHashSetName();
        }

        @Override
        public String getDatabasePath() throws TskCoreException {
            return SleuthkitJNI.getHashDatabasePath(handle);
        }
        
        public void setIndexing(boolean indexing){
            this.indexing = indexing; 
        }

        @Override
        public String getIndexPath() throws TskCoreException {
            return SleuthkitJNI.getHashDatabaseIndexPath(handle);
        }

        @Override
        public KnownFilesType getKnownFilesType() {
            return knownFilesType;
        }

        @Override
        public boolean getSearchDuringIngest() {
            return searchDuringIngest;
        }

        @Override
        void setSearchDuringIngest(boolean useForIngest) {
            this.searchDuringIngest = useForIngest;
        }

        @Override
        public boolean getSendIngestMessages() {
            return sendIngestMessages;
        }

        @Override
        void setSendIngestMessages(boolean showInboxMessages) {
            this.sendIngestMessages = showInboxMessages;
        }

        @Override
        public boolean isUpdateable() throws TskCoreException {
            return SleuthkitJNI.isUpdateableHashDatabase(this.handle);
        }

        @Override
        public void addHashes(Content content) throws TskCoreException {
            addHashes(content, null);
        }

        @Override
        public void addHashes(Content content, String comment) throws TskCoreException {
            // This only works for AbstractFiles and MD5 hashes at present. 
            assert content instanceof AbstractFile;
            if (content instanceof AbstractFile) {
                AbstractFile file = (AbstractFile) content;
                if (null != file.getMd5Hash()) {
                    SleuthkitJNI.addToHashDatabase(null, file.getMd5Hash(), null, null, comment, handle);
                }
            }
        }

        @Override
        public void addHashes(List<HashEntry> hashes) throws TskCoreException {
            SleuthkitJNI.addToHashDatabase(hashes, handle);
        }

        @Override
        public boolean lookupMD5Quick(Content content) throws TskCoreException {
            boolean result = false;
            assert content instanceof AbstractFile;
            if (content instanceof AbstractFile) {
                AbstractFile file = (AbstractFile) content;
                if (null != file.getMd5Hash()) {
                    result = SleuthkitJNI.lookupInHashDatabase(file.getMd5Hash(), handle);
                }
            }
            return result;
        }

        @Override
        public HashHitInfo lookupMD5(Content content) throws TskCoreException {
            HashHitInfo result = null;
            // This only works for AbstractFiles and MD5 hashes at present. 
            assert content instanceof AbstractFile;
            if (content instanceof AbstractFile) {
                AbstractFile file = (AbstractFile) content;
                if (null != file.getMd5Hash()) {
                    result = SleuthkitJNI.lookupInHashDatabaseVerbose(file.getMd5Hash(), handle);
                }
            }
            return result;
        }
        
        @Override
        boolean isValid() throws TskCoreException {
            return hasIndex();
        }

        boolean hasIndex() throws TskCoreException {
            return SleuthkitJNI.hashDatabaseHasLookupIndex(handle);
        }

        @Override
        public boolean hasIndexOnly() throws TskCoreException {
            return SleuthkitJNI.hashDatabaseIsIndexOnly(handle);
        }

        boolean canBeReIndexed() throws TskCoreException {
            return SleuthkitJNI.hashDatabaseCanBeReindexed(handle);
        }

        boolean isIndexing() {
            return indexing;
        }
        
        @Override
        public void firePropertyChange(String propertyName, Object oldValue, Object newValue){
            this.propertyChangeSupport.firePropertyChange(propertyName, oldValue, newValue);
        }

        private void close() throws TskCoreException {
            SleuthkitJNI.closeHashDatabase(handle);
        }
        
        @Override
        public String toString(){
            return getHashSetName();
        }
        

        @Override
        public int hashCode() {
            int code = 23;
            code = 47 * code + Integer.hashCode(handle);
            code = 47 * code + Objects.hashCode(this.hashSetName);
            code = 47 * code + Objects.hashCode(this.propertyChangeSupport);
            code = 47 * code + Objects.hashCode(this.knownFilesType);
            return code;
        }

        @Override
        public boolean equals(Object obj) {
            if (obj == null) {
                return false;
            }
            if (getClass() != obj.getClass()) {
                return false;
            }
            final SleuthkitHashSet other = (SleuthkitHashSet) obj;
            if (!Objects.equals(this.hashSetName, other.hashSetName)) {
                return false;
            }
            if (this.knownFilesType != other.knownFilesType) {
                return false;
            }
            return true;
        }
    }

    class CentralRepoHashSet extends HashDb{

        private static final long serialVersionUID = 1L;
        private final String hashSetName;
        private boolean searchDuringIngest;
        private boolean sendIngestMessages;
        private final HashDb.KnownFilesType knownFilesType;  
        private final int referenceSetID;
        private final String version;
        private String orgName;
        private final boolean readOnly;
        private final PropertyChangeSupport propertyChangeSupport = new PropertyChangeSupport(this);

        @Messages({"HashDbManager.CentralRepoHashDb.orgError=Error loading organization"})
        private CentralRepoHashSet(String hashSetName, String version, int referenceSetID, 
                boolean useForIngest, boolean sendHitMessages, HashDb.KnownFilesType knownFilesType, 
                boolean readOnly)
                throws TskCoreException{
            this.hashSetName = hashSetName;
            this.version = version;
            this.referenceSetID = referenceSetID;
            this.searchDuringIngest = useForIngest;
            this.sendIngestMessages = sendHitMessages;
            this.knownFilesType = knownFilesType;
            this.readOnly = readOnly;
            
            try{
                orgName = EamDb.getInstance().getReferenceSetOrganization(referenceSetID).getName();
            } catch (EamDbException ex){
                Logger.getLogger(SleuthkitHashSet.class.getName()).log(Level.SEVERE, "Error looking up central repository organization for reference set " + referenceSetID, ex); //NON-NLS
                orgName = Bundle.HashDbManager_CentralRepoHashDb_orgError();
            }
        }

        @Override
        public void addPropertyChangeListener(PropertyChangeListener pcl) {
            propertyChangeSupport.addPropertyChangeListener(pcl);
        }

        @Override
        public void removePropertyChangeListener(PropertyChangeListener pcl) {
            propertyChangeSupport.removePropertyChangeListener(pcl);
        }
        
        @Override
        public boolean hasIndexOnly() throws TskCoreException{
            return true;
        }

        @Override
        public String getHashSetName() {
            return hashSetName;
        }
        
        @Override
        public String getDisplayName(){
            if(! getVersion().isEmpty()){
                return getHashSetName() + " " + getVersion() + " (remote)";
            } else {
                return getHashSetName() + " (remote)";
            }
        }
        
        String getVersion(){
            return version;
        }
        
        String getOrgName(){
            return orgName;
        }
        
        int getReferenceSetID(){
            return referenceSetID;
        }

        @Override
        public String getDatabasePath() throws TskCoreException {
            return "";
        }

        @Override
        public String getIndexPath() throws TskCoreException {
            return "";
        }

        @Override
        public HashDb.KnownFilesType getKnownFilesType() {
            return knownFilesType;
        }

        @Override
        public boolean getSearchDuringIngest() {
            return searchDuringIngest;
        }

        @Override
        void setSearchDuringIngest(boolean useForIngest) {
            this.searchDuringIngest = useForIngest;
        }

        @Override
        public boolean getSendIngestMessages() {
            return sendIngestMessages;
        }

        @Override
        void setSendIngestMessages(boolean showInboxMessages) {
            this.sendIngestMessages = showInboxMessages;
        }

        @Override
        public boolean isUpdateable() throws TskCoreException {
            return (! readOnly);
        }

        @Override
        public void addHashes(Content content) throws TskCoreException {
            addHashes(content, null);
        }

        @Override
        public void addHashes(Content content, String comment) throws TskCoreException {
            // This only works for AbstractFiles and MD5 hashes at present. 
            assert content instanceof AbstractFile;
            if (content instanceof AbstractFile) {
                AbstractFile file = (AbstractFile) content;
                if (null != file.getMd5Hash()) {
                    TskData.FileKnown type;
                    if(knownFilesType.equals(HashDb.KnownFilesType.KNOWN_BAD)){
                        type = TskData.FileKnown.BAD;
                    } else {
                        type = TskData.FileKnown.KNOWN;
                    }
                    
                    try{
                        EamGlobalFileInstance fileInstance = new EamGlobalFileInstance(referenceSetID, file.getMd5Hash(),
                            type, comment);
                        EamDb.getInstance().addReferenceInstance(fileInstance,EamDb.getInstance().getCorrelationTypeById(CorrelationAttributeInstance.FILES_TYPE_ID));
                    } catch (EamDbException | CorrelationAttributeNormalizationException ex){
                        throw new TskCoreException("Error adding hashes to " + getDisplayName(), ex);   //NON-NLS
                    }
                }
            }
        }

        @Override
        public void addHashes(List<HashEntry> hashes) throws TskCoreException {
            Set<EamGlobalFileInstance> globalFileInstances = new HashSet<>();
            for(HashEntry hashEntry:hashes){
                TskData.FileKnown type;
                if(knownFilesType.equals(HashDb.KnownFilesType.KNOWN_BAD)){
                    type = TskData.FileKnown.BAD;
                } else {
                    type = TskData.FileKnown.KNOWN;
                }       
                try {
                    globalFileInstances.add(new EamGlobalFileInstance(referenceSetID, hashEntry.getMd5Hash(), type, hashEntry.getComment()));
                } catch (EamDbException | CorrelationAttributeNormalizationException ex){
                    throw new TskCoreException("Error adding hashes to " + getDisplayName(), ex);
                }
            }
            
            try{
                EamDb.getInstance().bulkInsertReferenceTypeEntries(globalFileInstances, 
                        EamDb.getInstance().getCorrelationTypeById(CorrelationAttributeInstance.FILES_TYPE_ID));
            } catch (EamDbException ex){
                throw new TskCoreException("Error adding hashes to " + getDisplayName(), ex);
            }
        }

        @Override
        public boolean lookupMD5Quick(Content content) throws TskCoreException {
            // This only works for AbstractFiles and MD5 hashes 
            assert content instanceof AbstractFile;
            if (content instanceof AbstractFile) {
                AbstractFile file = (AbstractFile) content;
                if (null != file.getMd5Hash()) {
                    try{
                        return EamDb.getInstance().isFileHashInReferenceSet(file.getMd5Hash(), this.referenceSetID);
                    } catch (EamDbException | CorrelationAttributeNormalizationException ex){
                        Logger.getLogger(SleuthkitHashSet.class.getName()).log(Level.SEVERE, "Error performing central reposiotry hash lookup for hash "
                                + file.getMd5Hash() + " in reference set " + referenceSetID, ex); //NON-NLS
                        throw new TskCoreException("Error performing central reposiotry hash lookup", ex);
                    }
                }
            }
            return false;
        }

        @Override
        public HashHitInfo lookupMD5(Content content) throws TskCoreException {
            HashHitInfo result = null;
            // This only works for AbstractFiles and MD5 hashes 
            assert content instanceof AbstractFile;
            if (content instanceof AbstractFile) {
                AbstractFile file = (AbstractFile) content;
                if (null != file.getMd5Hash()) {
                    try{
                        if(EamDb.getInstance().isFileHashInReferenceSet(file.getMd5Hash(), this.referenceSetID)){
                            // Make a bare-bones HashHitInfo for now
                            result = new HashHitInfo(file.getMd5Hash(), "", "");
                        }
                    } catch (EamDbException | CorrelationAttributeNormalizationException ex){
                        Logger.getLogger(SleuthkitHashSet.class.getName()).log(Level.SEVERE, "Error performing central reposiotry hash lookup for hash "
                                + file.getMd5Hash() + " in reference set " + referenceSetID, ex); //NON-NLS
                        throw new TskCoreException("Error performing central reposiotry hash lookup", ex);
                    }
                }
            }
            return result;
        }
        
        @Override
        boolean isValid() {
            if(! EamDb.isEnabled()) {
                return false;
            }
            try{
                return EamDb.getInstance().referenceSetIsValid(this.referenceSetID, this.hashSetName, this.version);
            } catch (EamDbException ex){
                Logger.getLogger(CentralRepoHashSet.class.getName()).log(Level.SEVERE, "Error validating hash set " + hashSetName, ex); //NON-NLS
                return false;
            }
        }
        
        @Override
        public void firePropertyChange(String propertyName, Object oldValue, Object newValue){
            this.propertyChangeSupport.firePropertyChange(propertyName, oldValue, newValue);
        }
        
        @Override
        public String toString(){
            return getDisplayName();
        }
        

        @Override
        public int hashCode() {
            int code = 23;
            code = 47 * code + Objects.hashCode(this.hashSetName);
            code = 47 * code + Objects.hashCode(this.version);
            code = 47 * code + Integer.hashCode(this.referenceSetID);
            code = 47 * code + Objects.hashCode(this.knownFilesType);
            return code;
        }

        @Override
        public boolean equals(Object obj) {
            if (obj == null) {
                return false;
            }
            if (getClass() != obj.getClass()) {
                return false;
            }
            final CentralRepoHashSet other = (CentralRepoHashSet) obj;
            if (!Objects.equals(this.hashSetName, other.hashSetName)) {
                return false;
            }
            if (!Objects.equals(this.version, other.version)) {
                return false;
            }
            if (this.knownFilesType != other.knownFilesType) {
                return false;
            }
            return true;
        }
    }    
    
    private class HashDbIndexer extends SwingWorker<Object, Void> {

        private ProgressHandle progress = null;
        private SleuthkitHashSet hashDb = null;

        HashDbIndexer(SleuthkitHashSet hashDb) {
            this.hashDb = hashDb;
        }

        @Override
        protected Object doInBackground() {
            hashDb.setIndexing(true);
            progress = ProgressHandle.createHandle(
                    NbBundle.getMessage(this.getClass(), "HashDbManager.progress.indexingHashSet", hashDb.getHashSetName()));
            progress.start();
            progress.switchToIndeterminate();
            try {
                SleuthkitJNI.createLookupIndexForHashDatabase(hashDb.getHandle());
            } catch (TskCoreException ex) {
                Logger.getLogger(HashDbIndexer.class.getName()).log(Level.SEVERE, "Error indexing hash set " + hashDb.getHashSetName(), ex); //NON-NLS
                JOptionPane.showMessageDialog(WindowManager.getDefault().getMainWindow(),
                        NbBundle.getMessage(this.getClass(),
                                "HashDbManager.dlgMsg.errorIndexingHashSet",
                                hashDb.getHashSetName()),
                        NbBundle.getMessage(this.getClass(), "HashDbManager.hashDbIndexingErr"),
                        JOptionPane.ERROR_MESSAGE);
            }
            return null;
        }

        @Override
        protected void done() {
            hashDb.setIndexing(false);
            progress.finish();

            // see if we got any errors
            try {
                get();
            } catch (InterruptedException | ExecutionException ex) {
                logger.log(Level.SEVERE, "Error creating index", ex); //NON-NLS
                MessageNotifyUtil.Notify.show(
                        NbBundle.getMessage(this.getClass(), "HashDbManager.errCreatingIndex.title"),
                        NbBundle.getMessage(this.getClass(), "HashDbManager.errCreatingIndex.msg", ex.getMessage()),
                        MessageNotifyUtil.MessageType.ERROR);
            } // catch and ignore if we were cancelled
            catch (java.util.concurrent.CancellationException ex) {
            }

            try {
                hashDb.firePropertyChange(SleuthkitHashSet.Event.INDEXING_DONE.toString(), null, hashDb);
                hashDb.firePropertyChange(HashDbManager.SetEvt.DB_INDEXED.toString(), null, hashDb.getHashSetName());
            } catch (Exception e) {
                logger.log(Level.SEVERE, "HashDbManager listener threw exception", e); //NON-NLS
                MessageNotifyUtil.Notify.show(
                        NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErr"),
                        NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErrorListeningToUpdatesMsg"),
                        MessageNotifyUtil.MessageType.ERROR);
            }
        }
    }
}