19 package org.sleuthkit.autopsy.modules.hashdatabase;
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.Collections;
25 import java.util.HashMap;
26 import java.util.List;
27 import java.util.concurrent.atomic.AtomicLong;
28 import java.util.logging.Level;
29 import org.openide.util.NbBundle;
30 import org.openide.util.NbBundle.Messages;
43 import org.
sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
45 import org.
sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
54 "HashDbIngestModule.noKnownBadHashDbSetMsg=No notable hash set.",
55 "HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn=Notable file search will not be executed.",
56 "HashDbIngestModule.noKnownHashDbSetMsg=No known hash set.",
57 "HashDbIngestModule.knownFileSearchWillNotExecuteWarn=Known file search will not be executed."
62 private static final int MAX_COMMENT_SIZE = 500;
66 private final HashLookupModuleSettings
settings;
67 private List<HashDb> knownBadHashSets =
new ArrayList<>();
68 private List<HashDb> knownHashSets =
new ArrayList<>();
70 private static final HashMap<Long, IngestJobTotals> totalsForIngestJobs =
new HashMap<>();
76 private AtomicLong totalKnownBadCount =
new AtomicLong(0);
77 private AtomicLong totalCalctime =
new AtomicLong(0);
78 private AtomicLong totalLookuptime =
new AtomicLong(0);
85 totalsForIngestJobs.put(ingestJobId, totals);
91 this.settings = settings;
96 jobId = context.getJobId();
97 if (!hashDbManager.verifyAllDatabasesLoadedCorrectly()) {
98 throw new IngestModuleException(
"Could not load all hash sets");
105 getTotalsForIngestJobs(jobId);
108 if (knownBadHashSets.isEmpty()) {
111 Bundle.HashDbIngestModule_noKnownBadHashDbSetMsg(),
112 Bundle.HashDbIngestModule_knownBadFileSearchWillNotExecuteWarn()));
115 if (knownHashSets.isEmpty()) {
118 Bundle.HashDbIngestModule_noKnownHashDbSetMsg(),
119 Bundle.HashDbIngestModule_knownFileSearchWillNotExecuteWarn()));
131 enabledHashSets.clear();
132 for (
HashDb db : allHashSets) {
133 if (settings.isHashSetEnabled(db)) {
136 enabledHashSets.add(db);
138 }
catch (TskCoreException ex) {
139 logger.log(Level.WARNING,
"Error getting index status for " + db.getDisplayName()+
" hash set", ex);
150 if ((file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) ||
151 file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) {
166 if ((knownHashSets.isEmpty()) && (knownBadHashSets.isEmpty()) && (!settings.shouldCalculateHashes())) {
174 String name = file.getName();
175 String md5Hash = file.getMd5Hash();
176 if (md5Hash == null || md5Hash.isEmpty()) {
178 long calcstart = System.currentTimeMillis();
179 md5Hash = HashUtility.calculateMd5Hash(file);
180 file.setMd5Hash(md5Hash);
181 long delta = (System.currentTimeMillis() - calcstart);
184 }
catch (IOException ex) {
185 logger.log(Level.WARNING,
"Error calculating hash of file " + name, ex);
188 NbBundle.getMessage(this.getClass(),
189 "HashDbIngestModule.fileReadErrorMsg",
191 NbBundle.getMessage(this.getClass(),
192 "HashDbIngestModule.calcHashValueErr",
199 boolean foundBad =
false;
201 for (
HashDb db : knownBadHashSets) {
203 long lookupstart = System.currentTimeMillis();
204 HashHitInfo hashInfo = db.lookupMD5(file);
205 if (null != hashInfo) {
209 file.setKnown(TskData.FileKnown.BAD);
211 String hashSetName = db.getDisplayName();
214 ArrayList<String> comments = hashInfo.getComments();
216 for (String c : comments) {
221 if (comment.length() > MAX_COMMENT_SIZE) {
222 comment = comment.substring(0, MAX_COMMENT_SIZE) +
"...";
227 postHashSetHitToBlackboard(file, md5Hash, hashSetName, comment, db.getSendIngestMessages());
229 long delta = (System.currentTimeMillis() - lookupstart);
232 }
catch (TskException ex) {
233 logger.log(Level.WARNING,
"Couldn't lookup notable hash for file " + name +
" - see sleuthkit log for details", ex);
236 NbBundle.getMessage(this.getClass(),
237 "HashDbIngestModule.hashLookupErrorMsg",
239 NbBundle.getMessage(this.getClass(),
240 "HashDbIngestModule.lookingUpKnownBadHashValueErr",
250 for (
HashDb db : knownHashSets) {
252 long lookupstart = System.currentTimeMillis();
253 if (db.lookupMD5Quick(file)) {
254 file.setKnown(TskData.FileKnown.KNOWN);
257 long delta = (System.currentTimeMillis() - lookupstart);
260 }
catch (TskException ex) {
261 logger.log(Level.WARNING,
"Couldn't lookup known hash for file " + name +
" - see sleuthkit log for details", ex);
264 NbBundle.getMessage(this.getClass(),
265 "HashDbIngestModule.hashLookupErrorMsg",
267 NbBundle.getMessage(this.getClass(),
268 "HashDbIngestModule.lookingUpKnownHashValueErr",
278 @Messages({
"HashDbIngestModule.indexError.message=Failed to index hashset hit artifact for keyword search."})
279 private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment,
boolean showInboxMessage) {
281 String MODULE_NAME = NbBundle.getMessage(
HashDbIngestModule.class,
"HashDbIngestModule.moduleName");
283 BlackboardArtifact badFile = abstractFile.newArtifact(ARTIFACT_TYPE.TSK_HASHSET_HIT);
284 Collection<BlackboardAttribute> attributes =
new ArrayList<>();
287 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, hashSetName));
288 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_HASH_MD5, MODULE_NAME, md5Hash));
289 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME, comment));
291 badFile.addAttributes(attributes);
297 logger.log(Level.SEVERE,
"Unable to index blackboard artifact " + badFile.getArtifactID(), ex);
299 Bundle.HashDbIngestModule_indexError_message(), badFile.getDisplayName());
302 if (showInboxMessage) {
303 StringBuilder detailsSb =
new StringBuilder();
305 detailsSb.append(
"<table border='0' cellpadding='4' width='280'>");
307 detailsSb.append(
"<tr>");
308 detailsSb.append(
"<th>")
309 .append(NbBundle.getMessage(
this.getClass(),
"HashDbIngestModule.postToBB.fileName"))
311 detailsSb.append(
"<td>")
312 .append(abstractFile.getName())
314 detailsSb.append(
"</tr>");
316 detailsSb.append(
"<tr>");
317 detailsSb.append(
"<th>")
318 .append(NbBundle.getMessage(
this.getClass(),
"HashDbIngestModule.postToBB.md5Hash"))
320 detailsSb.append(
"<td>").append(md5Hash).append(
"</td>");
321 detailsSb.append(
"</tr>");
323 detailsSb.append(
"<tr>");
324 detailsSb.append(
"<th>")
325 .append(NbBundle.getMessage(
this.getClass(),
"HashDbIngestModule.postToBB.hashsetName"))
327 detailsSb.append(
"<td>").append(hashSetName).append(
"</td>");
328 detailsSb.append(
"</tr>");
330 detailsSb.append(
"</table>");
333 NbBundle.getMessage(this.getClass(),
334 "HashDbIngestModule.postToBB.knownBadMsg",
335 abstractFile.getName()),
336 detailsSb.toString(),
337 abstractFile.getName() + md5Hash,
341 }
catch (TskException ex) {
342 logger.log(Level.WARNING,
"Error creating blackboard artifact", ex);
347 List<HashDb> knownBadHashSets, List<HashDb> knownHashSets) {
349 totalsForIngestJobs.remove(jobId);
351 if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty())) {
352 StringBuilder detailsSb =
new StringBuilder();
354 detailsSb.append(
"<table border='0' cellpadding='4' width='280'>");
356 detailsSb.append(
"<tr><td>")
357 .append(NbBundle.getMessage(
HashDbIngestModule.class,
"HashDbIngestModule.complete.knownBadsFound"))
359 detailsSb.append(
"<td>").append(jobTotals.
totalKnownBadCount.get()).append(
"</td></tr>");
361 detailsSb.append(
"<tr><td>")
362 .append(NbBundle.getMessage(
HashDbIngestModule.class,
"HashDbIngestModule.complete.totalCalcTime"))
363 .append(
"</td><td>").append(jobTotals.
totalCalctime.get()).append(
"</td></tr>\n");
364 detailsSb.append(
"<tr><td>")
365 .append(NbBundle.getMessage(
HashDbIngestModule.class,
"HashDbIngestModule.complete.totalLookupTime"))
366 .append(
"</td><td>").append(jobTotals.
totalLookuptime.get()).append(
"</td></tr>\n");
367 detailsSb.append(
"</table>");
369 detailsSb.append(
"<p>")
370 .append(NbBundle.getMessage(
HashDbIngestModule.class,
"HashDbIngestModule.complete.databasesUsed"))
371 .append(
"</p>\n<ul>");
372 for (
HashDb db : knownBadHashSets) {
373 detailsSb.append(
"<li>").append(db.getHashSetName()).append(
"</li>\n");
376 detailsSb.append(
"</ul>");
382 "HashDbIngestModule.complete.hashLookupResults"),
383 detailsSb.toString()));
390 postSummary(jobId, knownBadHashSets, knownHashSets);
synchronized long decrementAndGet(long jobId)
ProcessResult process(AbstractFile file)
static IngestMessage createDataMessage(String source, String subject, String detailsHtml, String uniqueKey, BlackboardArtifact data)
final HashLookupModuleSettings settings
static IngestMessage createErrorMessage(String source, String subject, String detailsHtml)
AtomicLong totalKnownBadCount
AtomicLong totalLookuptime
void startUp(org.sleuthkit.autopsy.ingest.IngestJobContext context)
synchronized long incrementAndGet(long jobId)
static IngestMessage createMessage(MessageType messageType, String source, String subject, String detailsHtml)
static synchronized IngestJobTotals getTotalsForIngestJobs(long ingestJobId)
void updateEnabledHashSets(List< HashDb > allHashSets, List< HashDb > enabledHashSets)
void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage)
synchronized List< HashDb > getKnownBadFileHashSets()
static synchronized HashDbManager getInstance()
void postMessage(final IngestMessage message)
void fireModuleDataEvent(ModuleDataEvent moduleDataEvent)
SleuthkitCase getSleuthkitCase()
Blackboard getBlackboard()
static synchronized void postSummary(long jobId, List< HashDb > knownBadHashSets, List< HashDb > knownHashSets)
static void error(String title, String message)
synchronized void indexArtifact(BlackboardArtifact artifact)
static Case getCurrentCase()
synchronized static Logger getLogger(String name)
static IngestMessage createWarningMessage(String source, String subject, String detailsHtml)
synchronized List< HashDb > getKnownFileHashSets()
static synchronized IngestServices getInstance()