19 package org.sleuthkit.autopsy.modules.hashdatabase;
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.Collections;
25 import java.util.HashMap;
26 import java.util.List;
27 import java.util.concurrent.atomic.AtomicLong;
28 import java.util.logging.Level;
29 import org.openide.util.NbBundle;
30 import org.openide.util.NbBundle.Messages;
46 import org.
sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
48 import org.
sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
60 "HashDbIngestModule.noKnownBadHashDbSetMsg=No notable hash set.",
61 "HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn=Notable file search will not be executed.",
62 "HashDbIngestModule.noKnownHashDbSetMsg=No known hash set.",
63 "HashDbIngestModule.knownFileSearchWillNotExecuteWarn=Known file search will not be executed."
68 private static final int MAX_COMMENT_SIZE = 500;
72 private final HashLookupModuleSettings
settings;
73 private final List<HashDb> knownBadHashSets =
new ArrayList<>();
74 private final List<HashDb> knownHashSets =
new ArrayList<>();
76 private static final HashMap<Long, IngestJobTotals> totalsForIngestJobs =
new HashMap<>();
85 private final AtomicLong totalKnownBadCount =
new AtomicLong(0);
86 private final AtomicLong totalCalctime =
new AtomicLong(0);
87 private final AtomicLong totalLookuptime =
new AtomicLong(0);
94 totalsForIngestJobs.put(ingestJobId, totals);
109 this.settings = settings;
115 jobId = context.getJobId();
116 if (!hashDbManager.verifyAllDatabasesLoadedCorrectly()) {
117 throw new IngestModuleException(
"Could not load all hash sets");
124 getTotalsForIngestJobs(jobId);
127 if (knownBadHashSets.isEmpty()) {
130 Bundle.HashDbIngestModule_noKnownBadHashDbSetMsg(),
131 Bundle.HashDbIngestModule_knownBadFileSearchWillNotExecuteWarn()));
134 if (knownHashSets.isEmpty()) {
137 Bundle.HashDbIngestModule_noKnownHashDbSetMsg(),
138 Bundle.HashDbIngestModule_knownFileSearchWillNotExecuteWarn()));
150 enabledHashSets.clear();
151 for (
HashDb db : allHashSets) {
152 if (settings.isHashSetEnabled(db)) {
155 enabledHashSets.add(db);
157 }
catch (TskCoreException ex) {
158 logger.log(Level.WARNING,
"Error getting index status for " + db.getDisplayName() +
" hash set", ex);
166 "HashDbIngestModule.dialogTitle.errorFindingArtifacts=Error Finding Artifacts: {0}",
168 "HashDbIngestModule.errorMessage.lookingForFileArtifacts=Error encountered while looking for existing artifacts for {0}."
175 logger.log(Level.SEVERE,
"Exception while getting open case.", ex);
180 if ((file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)
181 || file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) {
196 if ((knownHashSets.isEmpty()) && (knownBadHashSets.isEmpty()) && (!settings.shouldCalculateHashes())) {
204 String name = file.getName();
205 long fileId = file.getId();
206 String md5Hash = file.getMd5Hash();
207 if (md5Hash == null || md5Hash.isEmpty()) {
210 long calcstart = System.currentTimeMillis();
211 md5Hash = HashUtility.calculateMd5Hash(file);
212 if (file.getSize() > 0) {
216 if (file.getSize() < 1000000) {
223 file.setMd5Hash(md5Hash);
224 long delta = (System.currentTimeMillis() - calcstart);
227 }
catch (IOException ex) {
228 logger.log(Level.WARNING, String.format(
"Error calculating hash of file '%s' (id=%d).", name, fileId), ex);
231 NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.fileReadErrorMsg", name),
232 NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.calcHashValueErr", name)));
238 boolean foundBad =
false;
240 for (
HashDb db : knownBadHashSets) {
242 long lookupstart = System.currentTimeMillis();
243 HashHitInfo hashInfo = db.lookupMD5(file);
244 if (null != hashInfo) {
248 file.setKnown(TskData.FileKnown.BAD);
250 String hashSetName = db.getDisplayName();
253 ArrayList<String> comments = hashInfo.getComments();
255 for (String c : comments) {
260 if (comment.length() > MAX_COMMENT_SIZE) {
261 comment = comment.substring(0, MAX_COMMENT_SIZE) +
"...";
270 List<BlackboardAttribute> attributesList =
new ArrayList<>();
271 attributesList.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME,
HashLookupModuleFactory.getModuleName(), hashSetName));
273 org.
sleuthkit.datamodel.Blackboard tskBlackboard = skCase.getBlackboard();
274 if (tskBlackboard.artifactExists(file, BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT, attributesList) ==
false) {
275 postHashSetHitToBlackboard(file, md5Hash, hashSetName, comment, db.getSendIngestMessages());
277 }
catch (TskCoreException ex) {
278 logger.log(Level.SEVERE, String.format(
279 "A problem occurred while checking for existing artifacts for file '%s' (id=%d).", name, fileId), ex);
282 Bundle.HashDbIngestModule_dialogTitle_errorFindingArtifacts(name),
283 Bundle.HashDbIngestModule_errorMessage_lookingForFileArtifacts(name)));
287 long delta = (System.currentTimeMillis() - lookupstart);
290 }
catch (TskException ex) {
291 logger.log(Level.WARNING, String.format(
292 "Couldn't lookup notable hash for file '%s' (id=%d) - see sleuthkit log for details", name, fileId), ex);
295 NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.hashLookupErrorMsg", name),
296 NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.lookingUpKnownBadHashValueErr", name)));
305 for (
HashDb db : knownHashSets) {
307 long lookupstart = System.currentTimeMillis();
308 if (db.lookupMD5Quick(file)) {
309 file.setKnown(TskData.FileKnown.KNOWN);
312 long delta = (System.currentTimeMillis() - lookupstart);
315 }
catch (TskException ex) {
316 logger.log(Level.WARNING, String.format(
317 "Couldn't lookup known hash for file '%s' (id=%d) - see sleuthkit log for details", name, fileId), ex);
320 NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.hashLookupErrorMsg", name),
321 NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.lookingUpKnownHashValueErr", name)));
341 "HashDbIngestModule.indexError.message=Failed to index hashset hit artifact for keyword search."
343 private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment,
boolean showInboxMessage) {
346 BlackboardArtifact badFile = abstractFile.newArtifact(ARTIFACT_TYPE.TSK_HASHSET_HIT);
347 Collection<BlackboardAttribute> attributes =
new ArrayList<>();
350 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, moduleName, hashSetName));
351 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_HASH_MD5, moduleName, md5Hash));
352 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, moduleName, comment));
354 badFile.addAttributes(attributes);
360 logger.log(Level.SEVERE,
"Unable to index blackboard artifact " + badFile.getArtifactID(), ex);
362 Bundle.HashDbIngestModule_indexError_message(), badFile.getDisplayName());
365 if (showInboxMessage) {
366 StringBuilder detailsSb =
new StringBuilder();
368 detailsSb.append(
"<table border='0' cellpadding='4' width='280'>");
370 detailsSb.append(
"<tr>");
371 detailsSb.append(
"<th>")
372 .append(NbBundle.getMessage(
this.getClass(),
"HashDbIngestModule.postToBB.fileName"))
374 detailsSb.append(
"<td>")
375 .append(abstractFile.getName())
377 detailsSb.append(
"</tr>");
379 detailsSb.append(
"<tr>");
380 detailsSb.append(
"<th>")
381 .append(NbBundle.getMessage(
this.getClass(),
"HashDbIngestModule.postToBB.md5Hash"))
383 detailsSb.append(
"<td>").append(md5Hash).append(
"</td>");
384 detailsSb.append(
"</tr>");
386 detailsSb.append(
"<tr>");
387 detailsSb.append(
"<th>")
388 .append(NbBundle.getMessage(
this.getClass(),
"HashDbIngestModule.postToBB.hashsetName"))
390 detailsSb.append(
"<td>").append(hashSetName).append(
"</td>");
391 detailsSb.append(
"</tr>");
393 detailsSb.append(
"</table>");
396 NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.postToBB.knownBadMsg", abstractFile.getName()),
397 detailsSb.toString(),
398 abstractFile.getName() + md5Hash,
402 }
catch (TskException ex) {
403 logger.log(Level.WARNING,
"Error creating blackboard artifact", ex);
415 List<HashDb> knownBadHashSets, List<HashDb> knownHashSets) {
417 totalsForIngestJobs.remove(jobId);
419 if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty())) {
420 StringBuilder detailsSb =
new StringBuilder();
422 detailsSb.append(
"<table border='0' cellpadding='4' width='280'>");
424 detailsSb.append(
"<tr><td>")
425 .append(NbBundle.getMessage(
HashDbIngestModule.class,
"HashDbIngestModule.complete.knownBadsFound"))
427 detailsSb.append(
"<td>").append(jobTotals.
totalKnownBadCount.get()).append(
"</td></tr>");
429 detailsSb.append(
"<tr><td>")
430 .append(NbBundle.getMessage(
HashDbIngestModule.class,
"HashDbIngestModule.complete.totalCalcTime"))
431 .append(
"</td><td>").append(jobTotals.
totalCalctime.get()).append(
"</td></tr>\n");
432 detailsSb.append(
"<tr><td>")
433 .append(NbBundle.getMessage(
HashDbIngestModule.class,
"HashDbIngestModule.complete.totalLookupTime"))
434 .append(
"</td><td>").append(jobTotals.
totalLookuptime.get()).append(
"</td></tr>\n");
435 detailsSb.append(
"</table>");
437 detailsSb.append(
"<p>")
438 .append(NbBundle.getMessage(
HashDbIngestModule.class,
"HashDbIngestModule.complete.databasesUsed"))
439 .append(
"</p>\n<ul>");
440 for (
HashDb db : knownBadHashSets) {
441 detailsSb.append(
"<li>").append(db.getHashSetName()).append(
"</li>\n");
444 detailsSb.append(
"</ul>");
449 NbBundle.getMessage(
HashDbIngestModule.class,
"HashDbIngestModule.complete.hashLookupResults"),
450 detailsSb.toString()));
457 postSummary(jobId, knownBadHashSets, knownHashSets);
synchronized long decrementAndGet(long jobId)
ProcessResult process(AbstractFile file)
static IngestMessage createDataMessage(String source, String subject, String detailsHtml, String uniqueKey, BlackboardArtifact data)
final HashLookupModuleSettings settings
static IngestMessage createErrorMessage(String source, String subject, String detailsHtml)
void startUp(org.sleuthkit.autopsy.ingest.IngestJobContext context)
synchronized long incrementAndGet(long jobId)
static IngestMessage createMessage(MessageType messageType, String source, String subject, String detailsHtml)
static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId)
final AtomicLong totalLookuptime
final AtomicLong totalCalctime
void updateEnabledHashSets(List< HashDb > allHashSets, List< HashDb > enabledHashSets)
static TimingMetric getTimingMetric(String name)
void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage)
final SleuthkitCase skCase
synchronized List< HashDb > getKnownBadFileHashSets()
static synchronized HashDbManager getInstance()
final AtomicLong totalKnownBadCount
void postMessage(final IngestMessage message)
void fireModuleDataEvent(ModuleDataEvent moduleDataEvent)
SleuthkitCase getSleuthkitCase()
static void submitTimingMetric(TimingMetric metric)
Blackboard getBlackboard()
static synchronized void postSummary(long jobId, List< HashDb > knownBadHashSets, List< HashDb > knownHashSets)
static void error(String title, String message)
synchronized void indexArtifact(BlackboardArtifact artifact)
synchronized static Logger getLogger(String name)
static Case getCurrentCaseThrows()
static IngestMessage createWarningMessage(String source, String subject, String detailsHtml)
static void submitNormalizedTimingMetric(TimingMetric metric, long normalization)
synchronized List< HashDb > getKnownFileHashSets()
static synchronized IngestServices getInstance()