19 package org.sleuthkit.autopsy.modules.hashdatabase;
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collections;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.concurrent.atomic.AtomicLong;
27 import java.util.logging.Level;
28 import org.openide.util.NbBundle;
29 import org.openide.util.NbBundle.Messages;
42 import org.
sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
44 import org.
sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
53 "HashDbIngestModule.noKnownBadHashDbSetMsg=No known bad hash database set.",
54 "HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn=Known bad file search will not be executed.",
55 "HashDbIngestModule.noKnownHashDbSetMsg=No known hash database set.",
56 "HashDbIngestModule.knownFileSearchWillNotExecuteWarn=Known file search will not be executed."
61 private static final int MAX_COMMENT_SIZE = 500;
65 private final HashLookupModuleSettings
settings;
66 private List<HashDb> knownBadHashSets =
new ArrayList<>();
67 private List<HashDb> knownHashSets =
new ArrayList<>();
69 private static final HashMap<Long, IngestJobTotals> totalsForIngestJobs =
new HashMap<>();
75 private AtomicLong totalKnownBadCount =
new AtomicLong(0);
76 private AtomicLong totalCalctime =
new AtomicLong(0);
77 private AtomicLong totalLookuptime =
new AtomicLong(0);
84 totalsForIngestJobs.put(ingestJobId, totals);
90 this.settings = settings;
95 jobId = context.getJobId();
96 if (!hashDbManager.verifyAllDatabasesLoadedCorrectly()) {
97 throw new IngestModuleException(
"Could not load all hash databases");
104 getTotalsForIngestJobs(jobId);
107 if (knownBadHashSets.isEmpty()) {
110 Bundle.HashDbIngestModule_noKnownBadHashDbSetMsg(),
111 Bundle.HashDbIngestModule_knownBadFileSearchWillNotExecuteWarn()));
114 if (knownHashSets.isEmpty()) {
117 Bundle.HashDbIngestModule_noKnownHashDbSetMsg(),
118 Bundle.HashDbIngestModule_knownFileSearchWillNotExecuteWarn()));
130 enabledHashSets.clear();
131 for (
HashDb db : allHashSets) {
132 if (settings.isHashSetEnabled(db.getHashSetName())) {
135 enabledHashSets.add(db);
137 }
catch (TskCoreException ex) {
138 logger.log(Level.WARNING,
"Error getting index status for " + db.getHashSetName() +
" hash database", ex);
149 if (file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) {
164 if ((knownHashSets.isEmpty()) && (knownBadHashSets.isEmpty()) && (!settings.shouldCalculateHashes())) {
172 String name = file.getName();
173 String md5Hash = file.getMd5Hash();
174 if (md5Hash == null || md5Hash.isEmpty()) {
176 long calcstart = System.currentTimeMillis();
177 md5Hash = HashUtility.calculateMd5(file);
178 long delta = (System.currentTimeMillis() - calcstart);
181 }
catch (IOException ex) {
182 logger.log(Level.WARNING,
"Error calculating hash of file " + name, ex);
185 NbBundle.getMessage(this.getClass(),
186 "HashDbIngestModule.fileReadErrorMsg",
188 NbBundle.getMessage(this.getClass(),
189 "HashDbIngestModule.calcHashValueErr",
196 boolean foundBad =
false;
198 for (
HashDb db : knownBadHashSets) {
200 long lookupstart = System.currentTimeMillis();
201 HashHitInfo hashInfo = db.lookupMD5(file);
202 if (null != hashInfo) {
207 skCase.setKnown(file, TskData.FileKnown.BAD);
208 }
catch (TskException ex) {
209 logger.log(Level.WARNING,
"Couldn't set known bad state for file " + name +
" - see sleuthkit log for details", ex);
212 NbBundle.getMessage(this.getClass(),
213 "HashDbIngestModule.hashLookupErrorMsg",
215 NbBundle.getMessage(this.getClass(),
216 "HashDbIngestModule.settingKnownBadStateErr",
220 String hashSetName = db.getHashSetName();
223 ArrayList<String> comments = hashInfo.getComments();
225 for (String c : comments) {
230 if (comment.length() > MAX_COMMENT_SIZE) {
231 comment = comment.substring(0, MAX_COMMENT_SIZE) +
"...";
236 postHashSetHitToBlackboard(file, md5Hash, hashSetName, comment, db.getSendIngestMessages());
238 long delta = (System.currentTimeMillis() - lookupstart);
241 }
catch (TskException ex) {
242 logger.log(Level.WARNING,
"Couldn't lookup known bad hash for file " + name +
" - see sleuthkit log for details", ex);
245 NbBundle.getMessage(this.getClass(),
246 "HashDbIngestModule.hashLookupErrorMsg",
248 NbBundle.getMessage(this.getClass(),
249 "HashDbIngestModule.lookingUpKnownBadHashValueErr",
259 for (
HashDb db : knownHashSets) {
261 long lookupstart = System.currentTimeMillis();
262 if (db.lookupMD5Quick(file)) {
264 skCase.setKnown(file, TskData.FileKnown.KNOWN);
266 }
catch (TskException ex) {
267 logger.log(Level.WARNING,
"Couldn't set known state for file " + name +
" - see sleuthkit log for details", ex);
271 long delta = (System.currentTimeMillis() - lookupstart);
274 }
catch (TskException ex) {
275 logger.log(Level.WARNING,
"Couldn't lookup known hash for file " + name +
" - see sleuthkit log for details", ex);
278 NbBundle.getMessage(this.getClass(),
279 "HashDbIngestModule.hashLookupErrorMsg",
281 NbBundle.getMessage(this.getClass(),
282 "HashDbIngestModule.lookingUpKnownHashValueErr",
292 @Messages({
"HashDbIngestModule.indexError.message=Failed to index hashset hit artifact for keyword search."})
293 private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment,
boolean showInboxMessage) {
295 String MODULE_NAME = NbBundle.getMessage(
HashDbIngestModule.class,
"HashDbIngestModule.moduleName");
297 BlackboardArtifact badFile = abstractFile.newArtifact(ARTIFACT_TYPE.TSK_HASHSET_HIT);
300 BlackboardAttribute att2 =
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, hashSetName);
301 badFile.addAttribute(att2);
302 BlackboardAttribute att3 =
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_HASH_MD5, MODULE_NAME, md5Hash);
303 badFile.addAttribute(att3);
304 BlackboardAttribute att4 =
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME, comment);
305 badFile.addAttribute(att4);
311 logger.log(Level.SEVERE,
"Unable to index blackboard artifact " + badFile.getArtifactID(), ex);
313 Bundle.HashDbIngestModule_indexError_message(), badFile.getDisplayName());
316 if (showInboxMessage) {
317 StringBuilder detailsSb =
new StringBuilder();
319 detailsSb.append(
"<table border='0' cellpadding='4' width='280'>");
321 detailsSb.append(
"<tr>");
322 detailsSb.append(
"<th>")
323 .append(NbBundle.getMessage(
this.getClass(),
"HashDbIngestModule.postToBB.fileName"))
325 detailsSb.append(
"<td>")
326 .append(abstractFile.getName())
328 detailsSb.append(
"</tr>");
330 detailsSb.append(
"<tr>");
331 detailsSb.append(
"<th>")
332 .append(NbBundle.getMessage(
this.getClass(),
"HashDbIngestModule.postToBB.md5Hash"))
334 detailsSb.append(
"<td>").append(md5Hash).append(
"</td>");
335 detailsSb.append(
"</tr>");
337 detailsSb.append(
"<tr>");
338 detailsSb.append(
"<th>")
339 .append(NbBundle.getMessage(
this.getClass(),
"HashDbIngestModule.postToBB.hashsetName"))
341 detailsSb.append(
"<td>").append(hashSetName).append(
"</td>");
342 detailsSb.append(
"</tr>");
344 detailsSb.append(
"</table>");
347 NbBundle.getMessage(this.getClass(),
348 "HashDbIngestModule.postToBB.knownBadMsg",
349 abstractFile.getName()),
350 detailsSb.toString(),
351 abstractFile.getName() + md5Hash,
355 }
catch (TskException ex) {
356 logger.log(Level.WARNING,
"Error creating blackboard artifact", ex);
361 List<HashDb> knownBadHashSets, List<HashDb> knownHashSets) {
363 totalsForIngestJobs.remove(jobId);
365 if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty())) {
366 StringBuilder detailsSb =
new StringBuilder();
368 detailsSb.append(
"<table border='0' cellpadding='4' width='280'>");
370 detailsSb.append(
"<tr><td>")
371 .append(NbBundle.getMessage(
HashDbIngestModule.class,
"HashDbIngestModule.complete.knownBadsFound"))
373 detailsSb.append(
"<td>").append(jobTotals.
totalKnownBadCount.get()).append(
"</td></tr>");
375 detailsSb.append(
"<tr><td>")
376 .append(NbBundle.getMessage(
HashDbIngestModule.class,
"HashDbIngestModule.complete.totalCalcTime"))
377 .append(
"</td><td>").append(jobTotals.
totalCalctime.get()).append(
"</td></tr>\n");
378 detailsSb.append(
"<tr><td>")
379 .append(NbBundle.getMessage(
HashDbIngestModule.class,
"HashDbIngestModule.complete.totalLookupTime"))
380 .append(
"</td><td>").append(jobTotals.
totalLookuptime.get()).append(
"</td></tr>\n");
381 detailsSb.append(
"</table>");
383 detailsSb.append(
"<p>")
384 .append(NbBundle.getMessage(
HashDbIngestModule.class,
"HashDbIngestModule.complete.databasesUsed"))
385 .append(
"</p>\n<ul>");
386 for (
HashDb db : knownBadHashSets) {
387 detailsSb.append(
"<li>").append(db.getHashSetName()).append(
"</li>\n");
390 detailsSb.append(
"</ul>");
396 "HashDbIngestModule.complete.hashLookupResults"),
397 detailsSb.toString()));
404 postSummary(jobId, knownBadHashSets, knownHashSets);
synchronized long decrementAndGet(long jobId)
ProcessResult process(AbstractFile file)
static IngestMessage createDataMessage(String source, String subject, String detailsHtml, String uniqueKey, BlackboardArtifact data)
final HashLookupModuleSettings settings
static IngestMessage createErrorMessage(String source, String subject, String detailsHtml)
AtomicLong totalKnownBadCount
AtomicLong totalLookuptime
void startUp(org.sleuthkit.autopsy.ingest.IngestJobContext context)
synchronized long incrementAndGet(long jobId)
static IngestMessage createMessage(MessageType messageType, String source, String subject, String detailsHtml)
static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId)
void updateEnabledHashSets(List< HashDb > allHashSets, List< HashDb > enabledHashSets)
void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage)
synchronized List< HashDb > getKnownBadFileHashSets()
static synchronized HashDbManager getInstance()
void postMessage(final IngestMessage message)
void fireModuleDataEvent(ModuleDataEvent moduleDataEvent)
SleuthkitCase getSleuthkitCase()
Blackboard getBlackboard()
static synchronized void postSummary(long jobId, List< HashDb > knownBadHashSets, List< HashDb > knownHashSets)
static void error(String title, String message)
synchronized void indexArtifact(BlackboardArtifact artifact)
static Case getCurrentCase()
synchronized static Logger getLogger(String name)
static IngestMessage createWarningMessage(String source, String subject, String detailsHtml)
synchronized List< HashDb > getKnownFileHashSets()
static synchronized IngestServices getInstance()