19 package org.sleuthkit.autopsy.modules.hashdatabase;
21 import java.beans.PropertyChangeEvent;
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.HashSet;
26 import java.util.List;
28 import javax.swing.JFileChooser;
29 import javax.swing.filechooser.FileNameExtensionFilter;
30 import javax.xml.parsers.DocumentBuilder;
31 import javax.xml.parsers.DocumentBuilderFactory;
32 import javax.xml.parsers.ParserConfigurationException;
33 import org.openide.util.NbBundle;
36 import org.w3c.dom.Document;
37 import org.w3c.dom.Element;
38 import org.w3c.dom.NodeList;
39 import java.beans.PropertyChangeListener;
40 import java.beans.PropertyChangeSupport;
41 import java.util.concurrent.ExecutionException;
42 import java.util.logging.Level;
43 import javax.swing.JOptionPane;
44 import javax.swing.SwingWorker;
45 import org.apache.commons.io.FilenameUtils;
46 import org.apache.commons.io.FileUtils;
47 import org.netbeans.api.progress.ProgressHandle;
48 import org.netbeans.api.progress.ProgressHandleFactory;
76 private static final String
ENCODING =
"UTF-8";
84 PropertyChangeSupport changeSupport =
new PropertyChangeSupport(
HashDbManager.class);
94 DB_ADDED, DB_DELETED, DB_INDEXED
101 if (instance == null) {
108 changeSupport.addPropertyChangeListener(listener);
122 static String getHashDatabaseFileExtension() {
155 addExistingHashDatabaseInternal(hashSetName, path, searchDuringIngest, sendIngestMessages, knownFilesType);
156 }
catch (TskCoreException ex) {
157 throw new HashDbManagerException(ex.getMessage());
162 throw new HashDbManagerException(NbBundle.getMessage(
this.getClass(),
"HashDbManager.saveErrorExceptionMsg"));
190 if (!
new File(path).exists()) {
191 throw new HashDbManagerException(NbBundle.getMessage(
HashDbManager.class,
"HashDbManager.hashDbDoesNotExistExceptionMsg", path));
194 if (hashSetPaths.contains(path)) {
195 throw new HashDbManagerException(NbBundle.getMessage(
HashDbManager.class,
"HashDbManager.hashDbAlreadyAddedExceptionMsg", path));
198 if (hashSetNames.contains(hashSetName)) {
199 throw new HashDbManagerException(NbBundle.getMessage(
HashDbManager.class,
"HashDbManager.duplicateHashSetNameExceptionMsg", hashSetName));
202 return addHashDatabase(SleuthkitJNI.openHashDatabase(path), hashSetName, searchDuringIngest, sendIngestMessages, knownFilesType);
228 hashDb = addNewHashDatabaseInternal(hashSetName, path, searchDuringIngest, sendIngestMessages, knownFilesType);
229 }
catch (TskCoreException ex) {
230 throw new HashDbManagerException(ex.getMessage());
235 throw new HashDbManagerException(NbBundle.getMessage(
this.getClass(),
"HashDbManager.saveErrorExceptionMsg"));
261 synchronized HashDb addNewHashDatabaseInternal(String hashSetName, String path,
boolean searchDuringIngest,
boolean sendIngestMessages,
HashDb.
KnownFilesType knownFilesType) throws HashDbManagerException, TskCoreException {
262 File file =
new File(path);
264 throw new HashDbManagerException(NbBundle.getMessage(
HashDbManager.class,
"HashDbManager.hashDbFileExistsExceptionMsg", path));
266 if (!FilenameUtils.getExtension(file.getName()).equalsIgnoreCase(HASH_DATABASE_FILE_EXTENSON)) {
267 throw new HashDbManagerException(NbBundle.getMessage(
HashDbManager.class,
"HashDbManager.illegalHashDbFileNameExtensionMsg",
268 getHashDatabaseFileExtension()));
271 if (hashSetPaths.contains(path)) {
272 throw new HashDbManagerException(NbBundle.getMessage(
HashDbManager.class,
"HashDbManager.hashDbAlreadyAddedExceptionMsg", path));
275 if (hashSetNames.contains(hashSetName)) {
276 throw new HashDbManagerException(NbBundle.getMessage(
HashDbManager.class,
"HashDbManager.duplicateHashSetNameExceptionMsg", hashSetName));
279 return addHashDatabase(SleuthkitJNI.createHashDatabase(path), hashSetName, searchDuringIngest, sendIngestMessages, knownFilesType);
284 HashDb hashDb =
new HashDb(handle, hashSetName, searchDuringIngest, sendIngestMessages, knownFilesType);
294 if (!databasePath.equals(
"None")) {
295 hashSetPaths.add(databasePath);
297 if (!indexPath.equals(
"None")) {
298 hashSetPaths.add(indexPath);
303 knownHashSets.add(hashDb);
305 knownBadHashSets.add(hashDb);
310 changeSupport.firePropertyChange(
SetEvt.
DB_ADDED.toString(), null, hashSetName);
311 }
catch (Exception e) {
312 logger.log(Level.SEVERE,
"HashDbManager listener threw exception", e);
314 NbBundle.getMessage(
this.getClass(),
"HashDbManager.moduleErr"),
315 NbBundle.getMessage(
this.getClass(),
"HashDbManager.moduleErrorListeningToUpdatesMsg"),
321 synchronized void indexHashDatabase(
HashDb hashDb) {
323 HashDbIndexer creator =
new HashDbIndexer(hashDb);
331 if (null != hashDb) {
334 if (!indexPath.equals(
"None")) {
335 hashSetPaths.add(indexPath);
337 }
catch (TskCoreException ex) {
355 if (ingestIsRunning) {
356 throw new HashDbManagerException(NbBundle.getMessage(
this.getClass(),
"HashDbManager.ingestRunningExceptionMsg"));
358 removeHashDatabaseInternal(hashDb);
360 throw new HashDbManagerException(NbBundle.getMessage(
this.getClass(),
"HashDbManager.saveErrorExceptionMsg"));
372 synchronized void removeHashDatabaseInternal(
HashDb hashDb) {
378 knownHashSets.remove(hashDb);
379 knownBadHashSets.remove(hashDb);
380 hashSetNames.remove(hashSetName);
385 }
catch (TskCoreException ex) {
386 Logger.getLogger(
HashDbManager.class.getName()).log(Level.SEVERE,
"Error getting index path of " + hashDb.
getHashSetName() +
" hash database when removing the database", ex);
389 if (!hashDb.hasIndexOnly()) {
392 }
catch (TskCoreException ex) {
393 Logger.getLogger(
HashDbManager.class.getName()).log(Level.SEVERE,
"Error getting database path of " + hashDb.
getHashSetName() +
" hash database when removing the database", ex);
397 }
catch (TskCoreException ex) {
398 Logger.getLogger(
HashDbManager.class.getName()).log(Level.SEVERE,
"Error closing " + hashDb.
getHashSetName() +
" hash database when removing the database", ex);
403 changeSupport.firePropertyChange(SetEvt.DB_DELETED.toString(), null, hashSetName);
404 }
catch (Exception e) {
405 logger.log(Level.SEVERE,
"HashDbManager listener threw exception", e);
406 MessageNotifyUtil.Notify.show(
407 NbBundle.getMessage(
this.getClass(),
"HashDbManager.moduleErr"),
408 NbBundle.getMessage(
this.getClass(),
"HashDbManager.moduleErrorListeningToUpdatesMsg"),
409 MessageNotifyUtil.MessageType.ERROR);
420 List<HashDb> hashDbs =
new ArrayList<>();
421 hashDbs.addAll(knownHashSets);
422 hashDbs.addAll(knownBadHashSets);
432 List<HashDb> hashDbs =
new ArrayList<>();
433 hashDbs.addAll(knownHashSets);
443 List<HashDb> hashDbs =
new ArrayList<>();
444 hashDbs.addAll(knownBadHashSets);
456 return updateableDbs;
460 ArrayList<HashDb> updateableDbs =
new ArrayList<>();
461 for (
HashDb db : hashDbs) {
463 if (db.isUpdateable()) {
464 updateableDbs.add(db);
466 }
catch (TskCoreException ex) {
467 Logger.
getLogger(
HashDbManager.class.getName()).log(Level.SEVERE,
"Error checking updateable status of " + db.getHashSetName() +
" hash database", ex);
470 return updateableDbs;
479 synchronized boolean save() {
490 hashSetNames.clear();
491 hashSetPaths.clear();
499 for (
HashDb database : hashDatabases) {
502 }
catch (TskCoreException ex) {
506 hashDatabases.clear();
510 boolean success =
false;
511 DocumentBuilderFactory dbfac = DocumentBuilderFactory.newInstance();
513 DocumentBuilder docBuilder = dbfac.newDocumentBuilder();
514 Document doc = docBuilder.newDocument();
515 Element rootEl = doc.createElement(ROOT_ELEMENT);
516 doc.appendChild(rootEl);
522 }
catch (ParserConfigurationException ex) {
529 for (
HashDb db : hashDbs) {
534 if (db.hasIndexOnly()) {
535 path = db.getIndexPath();
537 path = db.getDatabasePath();
539 }
catch (TskCoreException ex) {
540 Logger.
getLogger(
HashDbManager.class.getName()).log(Level.SEVERE,
"Error getting path of hash database " + db.getHashSetName() +
", discarding from hash database configuration", ex);
544 Element setElement = doc.createElement(SET_ELEMENT);
545 setElement.setAttribute(SET_NAME_ATTRIBUTE, db.getHashSetName());
546 setElement.setAttribute(SET_TYPE_ATTRIBUTE, db.getKnownFilesType().toString());
547 setElement.setAttribute(SEARCH_DURING_INGEST_ATTRIBUTE, Boolean.toString(db.getSearchDuringIngest()));
548 setElement.setAttribute(SEND_INGEST_MESSAGES_ATTRIBUTE, Boolean.toString(db.getSendIngestMessages()));
549 Element pathElement = doc.createElement(PATH_ELEMENT);
550 pathElement.setTextContent(path);
551 setElement.appendChild(pathElement);
552 rootEl.appendChild(setElement);
557 File f =
new File(configFilePath);
558 return f.exists() && f.canRead() && f.canWrite();
562 boolean updatedSchema =
false;
571 Element root = doc.getDocumentElement();
578 NodeList setsNList = root.getElementsByTagName(SET_ELEMENT);
579 int numSets = setsNList.getLength();
586 String attributeErrorMessage =
" attribute was not set for hash_set at index {0}, cannot make instance of HashDb class";
587 String elementErrorMessage =
" element was not set for hash_set at index {0}, cannot make instance of HashDb class";
588 for (
int i = 0; i < numSets; ++i) {
589 Element setEl = (Element) setsNList.item(i);
591 String hashSetName = setEl.getAttribute(SET_NAME_ATTRIBUTE);
592 if (hashSetName.isEmpty()) {
598 if (hashSetNames.contains(hashSetName)) {
600 String newHashSetName;
603 newHashSetName = hashSetName + suffix;
604 }
while (hashSetNames.contains(newHashSetName));
605 JOptionPane.showMessageDialog(null,
606 NbBundle.getMessage(
this.getClass(),
607 "HashDbManager.replacingDuplicateHashsetNameMsg",
608 hashSetName, newHashSetName),
609 NbBundle.getMessage(
this.getClass(),
"HashDbManager.openHashDbErr"),
610 JOptionPane.ERROR_MESSAGE);
611 hashSetName = newHashSetName;
614 String knownFilesType = setEl.getAttribute(SET_TYPE_ATTRIBUTE);
615 if (knownFilesType.isEmpty()) {
621 if (knownFilesType.equals(
"NSRL")) {
623 updatedSchema =
true;
626 final String searchDuringIngest = setEl.getAttribute(SEARCH_DURING_INGEST_ATTRIBUTE);
627 if (searchDuringIngest.isEmpty()) {
631 Boolean seearchDuringIngestFlag = Boolean.parseBoolean(searchDuringIngest);
633 final String sendIngestMessages = setEl.getAttribute(SEND_INGEST_MESSAGES_ATTRIBUTE);
634 if (searchDuringIngest.isEmpty()) {
638 Boolean sendIngestMessagesFlag = Boolean.parseBoolean(sendIngestMessages);
641 NodeList pathsNList = setEl.getElementsByTagName(PATH_ELEMENT);
642 if (pathsNList.getLength() > 0) {
643 Element pathEl = (Element) pathsNList.item(0);
646 String legacyPathNumber = pathEl.getAttribute(LEGACY_PATH_NUMBER_ATTRIBUTE);
647 if (null != legacyPathNumber && !legacyPathNumber.isEmpty()) {
648 updatedSchema =
true;
651 dbPath = pathEl.getTextContent();
652 if (dbPath.isEmpty()) {
662 if (null != dbPath) {
664 addExistingHashDatabaseInternal(hashSetName, dbPath, seearchDuringIngestFlag, sendIngestMessagesFlag,
HashDb.
KnownFilesType.valueOf(knownFilesType));
665 }
catch (HashDbManagerException | TskCoreException ex) {
667 JOptionPane.showMessageDialog(null,
668 NbBundle.getMessage(
this.getClass(),
669 "HashDbManager.unableToOpenHashDbMsg", dbPath),
670 NbBundle.getMessage(
this.getClass(),
"HashDbManager.openHashDbErr"),
671 JOptionPane.ERROR_MESSAGE);
674 Logger.
getLogger(
HashDbManager.class.getName()).log(Level.WARNING,
"No valid path for hash_set at index {0}, cannot make instance of HashDb class", i);
679 String backupFilePath = configFilePath +
".v1_backup";
680 String messageBoxTitle = NbBundle.getMessage(this.getClass(),
681 "HashDbManager.msgBoxTitle.confFileFmtChanged");
682 String baseMessage = NbBundle.getMessage(this.getClass(),
683 "HashDbManager.baseMessage.updatedFormatHashDbConfig");
685 FileUtils.copyFile(
new File(configFilePath),
new File(backupFilePath));
686 JOptionPane.showMessageDialog(null,
687 NbBundle.getMessage(
this.getClass(),
688 "HashDbManager.savedBackupOfOldConfigMsg",
689 baseMessage, backupFilePath),
691 JOptionPane.INFORMATION_MESSAGE);
692 }
catch (IOException ex) {
693 Logger.
getLogger(
HashDbManager.class.getName()).log(Level.WARNING,
"Failed to save backup of old format configuration file to " + backupFilePath, ex);
694 JOptionPane.showMessageDialog(null, baseMessage, messageBoxTitle, JOptionPane.INFORMATION_MESSAGE);
705 File database =
new File(configuredPath);
706 if (database.exists()) {
707 return configuredPath;
711 String newPath = null;
712 if (JOptionPane.showConfirmDialog(null,
713 NbBundle.getMessage(
this.getClass(),
"HashDbManager.dlgMsg.dbNotFoundAtLoc",
714 hashSetName, configuredPath),
715 NbBundle.getMessage(
this.getClass(),
"HashDbManager.dlgTitle.MissingDb"),
716 JOptionPane.YES_NO_OPTION) == JOptionPane.YES_OPTION) {
718 if (null != newPath && !newPath.isEmpty()) {
719 database =
new File(newPath);
720 if (!database.exists()) {
729 String filePath = null;
730 JFileChooser fc =
new JFileChooser();
731 fc.setDragEnabled(
false);
732 fc.setFileSelectionMode(JFileChooser.FILES_ONLY);
733 String[] EXTENSION =
new String[]{
"txt",
"idx",
"hash",
"Hash",
"kdb"};
734 FileNameExtensionFilter filter =
new FileNameExtensionFilter(
735 NbBundle.getMessage(
this.getClass(),
"HashDbManager.fileNameExtensionFilter.title"), EXTENSION);
736 fc.setFileFilter(filter);
737 fc.setMultiSelectionEnabled(
false);
738 if (fc.showOpenDialog(null) == JFileChooser.APPROVE_OPTION) {
739 File f = fc.getSelectedFile();
741 filePath = f.getCanonicalPath();
742 }
catch (IOException ex) {
766 this.displayName = displayName;
770 return this.displayName;
789 private HashDb(
int handle, String hashSetName,
boolean useForIngest,
boolean sendHitMessages,
KnownFilesType knownFilesType) {
792 this.searchDuringIngest = useForIngest;
793 this.sendIngestMessages = sendHitMessages;
795 this.indexing =
false;
802 propertyChangeSupport.addPropertyChangeListener(pcl);
809 propertyChangeSupport.removePropertyChangeListener(pcl);
817 return SleuthkitJNI.getHashDatabasePath(handle);
821 return SleuthkitJNI.getHashDatabaseIndexPath(handle);
832 void setSearchDuringIngest(
boolean useForIngest) {
833 this.searchDuringIngest = useForIngest;
840 void setSendIngestMessages(
boolean showInboxMessages) {
841 this.sendIngestMessages = showInboxMessages;
850 return SleuthkitJNI.isUpdateableHashDatabase(this.handle);
861 public void addHashes(Content content)
throws TskCoreException {
875 public void addHashes(Content content, String comment)
throws TskCoreException {
877 assert content instanceof AbstractFile;
878 if (content instanceof AbstractFile) {
879 AbstractFile file = (AbstractFile) content;
880 if (null != file.getMd5Hash()) {
881 SleuthkitJNI.addToHashDatabase(null, file.getMd5Hash(), null, null, comment,
handle);
893 public void addHashes(List<HashEntry> hashes)
throws TskCoreException {
894 SleuthkitJNI.addToHashDatabase(hashes, handle);
907 boolean result =
false;
908 assert content instanceof AbstractFile;
909 if (content instanceof AbstractFile) {
910 AbstractFile file = (AbstractFile) content;
911 if (null != file.getMd5Hash()) {
912 result = SleuthkitJNI.lookupInHashDatabase(file.getMd5Hash(),
handle);
927 public HashHitInfo
lookupMD5(Content content)
throws TskCoreException {
928 HashHitInfo result = null;
930 assert content instanceof AbstractFile;
931 if (content instanceof AbstractFile) {
932 AbstractFile file = (AbstractFile) content;
933 if (null != file.getMd5Hash()) {
934 result = SleuthkitJNI.lookupInHashDatabaseVerbose(file.getMd5Hash(),
handle);
940 boolean hasIndex() throws TskCoreException {
941 return SleuthkitJNI.hashDatabaseHasLookupIndex(handle);
944 boolean hasIndexOnly() throws TskCoreException {
945 return SleuthkitJNI.hashDatabaseIsIndexOnly(handle);
948 boolean canBeReIndexed() throws TskCoreException {
949 return SleuthkitJNI.hashDatabaseCanBeReindexed(handle);
952 boolean isIndexing() {
956 private void close() throws TskCoreException {
957 SleuthkitJNI.closeHashDatabase(handle);
978 progress = ProgressHandleFactory.createHandle(
979 NbBundle.getMessage(
this.getClass(),
"HashDbManager.progress.indexingHashSet", hashDb.
hashSetName));
981 progress.switchToIndeterminate();
983 SleuthkitJNI.createLookupIndexForHashDatabase(hashDb.
handle);
984 }
catch (TskCoreException ex) {
986 JOptionPane.showMessageDialog(null,
987 NbBundle.getMessage(
this.getClass(),
988 "HashDbManager.dlgMsg.errorIndexingHashSet",
990 NbBundle.getMessage(
this.getClass(),
"HashDbManager.hashDbIndexingErr"),
991 JOptionPane.ERROR_MESSAGE);
1004 }
catch (InterruptedException | ExecutionException ex) {
1005 logger.log(Level.SEVERE,
"Error creating index", ex);
1007 NbBundle.getMessage(
this.getClass(),
"HashDbManager.errCreatingIndex.title"),
1008 NbBundle.getMessage(
this.getClass(),
"HashDbManager.errCreatingIndex.msg", ex.getMessage()),
1011 catch (java.util.concurrent.CancellationException ex) {
1017 }
catch (Exception e) {
1018 logger.log(Level.SEVERE,
"HashDbManager listener threw exception", e);
1020 NbBundle.getMessage(
this.getClass(),
"HashDbManager.moduleErr"),
1021 NbBundle.getMessage(
this.getClass(),
"HashDbManager.moduleErrorListeningToUpdatesMsg"),
boolean getSendIngestMessages()
Set< String > hashSetPaths
static final Logger logger
static final String HASH_DATABASE_FILE_EXTENSON
void addHashes(Content content, String comment)
static synchronized IngestManager getInstance()
static< T > Document loadDoc(Class< T > clazz, String xmlPath)
boolean getSearchDuringIngest()
synchronized void addPropertyChangeListener(PropertyChangeListener listener)
HashDb addExistingHashDatabase(String hashSetName, String path, boolean searchDuringIngest, boolean sendIngestMessages, HashDb.KnownFilesType knownFilesType)
boolean hashSetsConfigurationFileExists()
HashDbManagerException(String message)
static final String LEGACY_PATH_NUMBER_ATTRIBUTE
final PropertyChangeSupport propertyChangeSupport
HashDb addHashDatabase(int handle, String hashSetName, boolean searchDuringIngest, boolean sendIngestMessages, HashDb.KnownFilesType knownFilesType)
boolean readHashSetsConfigurationFromDisk()
KnownFilesType knownFilesType
boolean isIngestRunning()
static void writeHashDbsToDisk(Document doc, Element rootEl, List< HashDb > hashDbs)
static final String SET_NAME_ATTRIBUTE
List< HashDb > getUpdateableHashSets(List< HashDb > hashDbs)
static final String SET_TYPE_ATTRIBUTE
List< HashDb > knownHashSets
static final String ENCODING
HashDb addNewHashDatabase(String hashSetName, String path, boolean searchDuringIngest, boolean sendIngestMessages, HashDb.KnownFilesType knownFilesType)
boolean writeHashSetConfigurationToDisk()
static final String PATH_ELEMENT
synchronized void removeHashDatabase(HashDb hashDb)
KnownFilesType(String displayName)
void addPropertyChangeListener(PropertyChangeListener pcl)
List< HashDb > knownBadHashSets
final String configFilePath
synchronized List< HashDb > getKnownBadFileHashSets()
static HashDbManager instance
Set< String > hashSetNames
static synchronized HashDbManager getInstance()
static final String SEARCH_DURING_INGEST_ATTRIBUTE
String getValidFilePath(String hashSetName, String configuredPath)
void addHashes(List< HashEntry > hashes)
void addHashes(Content content)
HashHitInfo lookupMD5(Content content)
synchronized List< HashDb > getUpdateableHashSets()
synchronized List< HashDb > getAllHashSets()
static final String SET_ELEMENT
static final String SEND_INGEST_MESSAGES_ATTRIBUTE
void propertyChange(PropertyChangeEvent event)
boolean sendIngestMessages
void closeHashDatabases(List< HashDb > hashDatabases)
boolean lookupMD5Quick(Content content)
boolean searchDuringIngest
synchronized void loadLastSavedConfiguration()
synchronized static Logger getLogger(String name)
static void show(String title, String message, MessageType type, ActionListener actionListener)
static final String XSD_FILE_NAME
KnownFilesType getKnownFilesType()
static final String CONFIG_FILE_NAME
static final String ROOT_ELEMENT
synchronized List< HashDb > getKnownFileHashSets()
void removePropertyChangeListener(PropertyChangeListener pcl)
HashDb(int handle, String hashSetName, boolean useForIngest, boolean sendHitMessages, KnownFilesType knownFilesType)
static< T > boolean saveDoc(Class< T > clazz, String xmlPath, String encoding, final Document doc)