19 package org.sleuthkit.autopsy.keywordsearch;
21 import java.util.ArrayList;
22 import java.util.Collection;
23 import java.util.HashMap;
24 import java.util.List;
26 import java.util.Map.Entry;
27 import java.util.Timer;
28 import java.util.TimerTask;
29 import java.util.concurrent.CancellationException;
30 import java.util.concurrent.ExecutionException;
31 import java.util.concurrent.atomic.AtomicLong;
32 import java.util.logging.Level;
33 import javax.swing.SwingUtilities;
34 import javax.swing.SwingWorker;
35 import org.netbeans.api.progress.aggregate.AggregateProgressFactory;
36 import org.netbeans.api.progress.aggregate.AggregateProgressHandle;
37 import org.netbeans.api.progress.aggregate.ProgressContributor;
38 import org.openide.util.Cancellable;
39 import org.openide.util.NbBundle;
40 import org.openide.util.NbBundle.Messages;
62 private Map<Long, SearchJobInfo>
jobs =
new HashMap<>();
66 updateTimer =
new Timer(NbBundle.getMessage(
this.getClass(),
"SearchRunner.updateTimer.title.text"),
true);
74 if (instance == null) {
90 public synchronized void startJob(
long jobId,
long dataSourceId, List<String> keywordListNames) {
91 if (jobs.containsKey(jobId) ==
false) {
92 logger.log(Level.INFO,
"Adding job {0}", jobId);
94 jobs.put(jobId, jobData);
98 jobs.get(jobId).incrementModuleReferenceCount();
101 if ((jobs.size() > 0) && (updateTimerRunning ==
false)) {
102 final long updateIntervalMs = ((long) KeywordSearchSettings.getUpdateFrequency().getTime()) * 60 * 1000;
103 updateTimer.scheduleAtFixedRate(
new UpdateTimerTask(), updateIntervalMs, updateIntervalMs);
104 updateTimerRunning =
true;
116 boolean readyForFinalSearch =
false;
117 synchronized (
this) {
118 job = jobs.get(jobId);
126 readyForFinalSearch =
true;
130 if (readyForFinalSearch) {
143 logger.log(Level.INFO,
"Stopping job {0}", jobId);
147 synchronized (
this) {
148 job = jobs.get(jobId);
155 if ((currentSearcher != null) && (!currentSearcher.isDone())) {
156 currentSearcher.cancel(
true);
170 for (String listName : keywordListNames) {
171 logger.log(Level.INFO,
"Adding keyword list {0} to all jobs", listName);
173 j.addKeywordListName(listName);
189 logger.log(Level.WARNING,
"Error executing Solr query to check number of indexed files: ", ex);
201 logger.log(Level.INFO,
"Running final search for jobid {0}", job.
getJobId());
209 finalSearcher.execute();
214 }
catch (InterruptedException | ExecutionException ex) {
215 logger.log(Level.WARNING,
"Job {1} final search thread failed: {2}",
new Object[]{job.getJobId(), ex});
230 if (jobs.isEmpty()) {
232 updateTimerRunning =
false;
240 for (Entry<Long, SearchJobInfo> j : jobs.entrySet()) {
270 public SearchJobInfo(
long jobId,
long dataSourceId, List<String> keywordListNames) {
274 currentResults =
new HashMap<>();
275 workerRunning =
false;
292 if (!keywordListNames.contains(keywordListName)) {
293 keywordListNames.add(keywordListName);
298 return currentResults.get(k);
302 currentResults.put(k, resultsIDs);
310 workerRunning = flag;
322 moduleReferenceCount.incrementAndGet();
326 return moduleReferenceCount.decrementAndGet();
336 while (workerRunning) {
337 finalSearchLock.wait();
347 workerRunning =
false;
348 finalSearchLock.notify();
359 private final class Searcher extends SwingWorker<Object, Void> {
376 keywords =
new ArrayList<>();
377 keywordToList =
new HashMap<>();
378 keywordLists =
new ArrayList<>();
388 @Messages(
"SearchRunner.query.exception.msg=Error performing query:")
390 final String displayName = NbBundle.getMessage(this.getClass(),
"KeywordSearchIngestModule.doInBackGround.displayName")
391 + (finalRun ? (
" - " + NbBundle.getMessage(this.getClass(),
"KeywordSearchIngestModule.doInBackGround.finalizeMsg")) :
"");
392 final String pgDisplayName = displayName + (
" (" + NbBundle.getMessage(this.getClass(),
"KeywordSearchIngestModule.doInBackGround.pendingMsg") +
")");
393 progressGroup = AggregateProgressFactory.createSystemHandle(pgDisplayName, null,
new Cancellable() {
395 public boolean cancel() {
396 logger.log(Level.INFO,
"Cancelling the searcher by user.");
397 if (progressGroup != null) {
398 progressGroup.setDisplayName(displayName +
" " + NbBundle.getMessage(
this.getClass(),
"SearchRunner.doInBackGround.cancelMsg"));
406 ProgressContributor[] subProgresses =
new ProgressContributor[keywords.size()];
408 for (Keyword keywordQuery : keywords) {
409 subProgresses[i] = AggregateProgressFactory.createProgressContributor(keywordQuery.getSearchTerm());
410 progressGroup.addContributor(subProgresses[i]);
414 progressGroup.start();
419 progressGroup.setDisplayName(displayName);
421 int keywordsSearched = 0;
423 for (Keyword keywordQuery : keywords) {
424 if (this.isCancelled()) {
425 logger.log(Level.INFO,
"Cancel detected, bailing before new keyword processed: {0}", keywordQuery.getSearchTerm());
429 final String queryStr = keywordQuery.getSearchTerm();
430 final KeywordList list = keywordToList.get(queryStr);
434 if (keywordsSearched > 0) {
435 subProgresses[keywordsSearched - 1].finish();
438 KeywordSearchQuery keywordSearchQuery = null;
440 boolean isRegex = !keywordQuery.searchTermIsLiteral();
442 keywordSearchQuery =
new TermsComponentQuery(list, keywordQuery);
444 keywordSearchQuery =
new LuceneQuery(list, keywordQuery);
445 keywordSearchQuery.escape();
451 final KeywordQueryFilter dataSourceFilter =
new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, job.
getDataSourceId());
452 keywordSearchQuery.addFilter(dataSourceFilter);
454 QueryResults queryResults;
458 queryResults = keywordSearchQuery.performQuery();
460 logger.log(Level.SEVERE,
"Error performing query: " + keywordQuery.getSearchTerm(), ex);
466 }
catch (CancellationException e) {
467 logger.log(Level.INFO,
"Cancel detected, bailing during keyword query: {0}", keywordQuery.getSearchTerm());
475 if (!newResults.getKeywords().isEmpty()) {
479 Collection<BlackboardArtifact> newArtifacts =
new ArrayList<>();
482 int totalUnits = newResults.getKeywords().size();
483 subProgresses[keywordsSearched].start(totalUnits);
484 int unitProgress = 0;
485 String queryDisplayStr = keywordQuery.getSearchTerm();
486 if (queryDisplayStr.length() > 50) {
487 queryDisplayStr = queryDisplayStr.substring(0, 49) +
"...";
489 subProgresses[keywordsSearched].progress(list.getName() +
": " + queryDisplayStr, unitProgress);
492 newArtifacts = newResults.writeAllHitsToBlackBoard(null, subProgresses[keywordsSearched],
this, list.getIngestMessages());
497 subProgresses[keywordsSearched].progress(
"");
504 catch (Exception ex) {
505 logger.log(Level.WARNING,
"searcher exception occurred", ex);
511 logger.log(Level.INFO,
"Searcher took to run: {0} secs.", stopWatch.
getElapsedTimeSecs());
526 }
catch (InterruptedException | ExecutionException e) {
527 logger.log(Level.SEVERE,
"Error performing keyword search: " + e.getMessage());
529 NbBundle.getMessage(this.getClass(),
530 "SearchRunner.Searcher.done.err.msg"), e.getMessage()));
532 catch (java.util.concurrent.CancellationException ex) {
540 XmlKeywordSearchList loader = XmlKeywordSearchList.getCurrent();
543 keywordToList.clear();
544 keywordLists.clear();
546 for (String name : keywordListNames) {
548 keywordLists.add(list);
549 for (Keyword k : list.getKeywords()) {
551 keywordToList.put(k.getSearchTerm(), list);
562 SwingUtilities.invokeLater(
new Runnable() {
565 progressGroup.finish();
574 QueryResults newResults =
new QueryResults(queryResult.getQuery(), queryResult.getKeywordList());
576 for (Keyword keyword : queryResult.getKeywords()) {
577 List<KeywordHit> queryTermResults = queryResult.getResults(keyword);
580 List<Long> queryTermResultsIDs =
new ArrayList<>();
581 for (KeywordHit ch : queryTermResults) {
582 queryTermResultsIDs.add(ch.getSolrObjectId());
586 if (curTermResults == null) {
588 newResults.addResult(keyword, queryTermResults);
591 for (KeywordHit res : queryTermResults) {
592 if (!curTermResults.contains(res.getSolrObjectId())) {
594 List<KeywordHit> newResultsFs = newResults.getResults(keyword);
595 if (newResultsFs == null) {
596 newResultsFs =
new ArrayList<>();
597 newResults.addResult(keyword, newResultsFs);
599 newResultsFs.add(res);
600 curTermResults.add(res.getSolrObjectId());
synchronized List< String > getKeywordListNames()
int queryNumIndexedFiles()
SearchJobInfo(long jobId, long dataSourceId, List< String > keywordListNames)
long getElapsedTimeSecs()
synchronized void addKeywordResults(Keyword k, List< Long > resultsIDs)
AggregateProgressHandle progressGroup
static IngestMessage createErrorMessage(String source, String subject, String detailsHtml)
static void fireNumIndexedFilesChange(Integer oldNum, Integer newNum)
synchronized void addKeywordListName(String keywordListName)
volatile boolean workerRunning
List< KeywordList > keywordLists
long decrementModuleReferenceCount()
AtomicLong moduleReferenceCount
synchronized void startJob(long jobId, long dataSourceId, List< String > keywordListNames)
static synchronized Server getServer()
boolean isWorkerRunning()
void doFinalSearch(SearchJobInfo job)
static final Logger logger
synchronized void setCurrentSearcher(SearchRunner.Searcher searchRunner)
final Object finalSearchLock
List< String > keywordListNames
synchronized List< Long > currentKeywordResults(Keyword k)
Map< Keyword, List< Long > > currentResults
Map< String, KeywordList > keywordToList
static synchronized SearchRunner getInstance()
synchronized void addKeywordListsToAllJobs(List< String > keywordListNames)
void postMessage(final IngestMessage message)
List< String > keywordListNames
Map< Long, SearchJobInfo > jobs
static SearchRunner instance
void incrementModuleReferenceCount()
void waitForCurrentWorker()
static void error(String title, String message)
synchronized static Logger getLogger(String name)
QueryResults filterResults(QueryResults queryResult)
SearchRunner.Searcher currentSearcher
static Ingester getIngester()
volatile boolean updateTimerRunning
synchronized SearchRunner.Searcher getCurrentSearcher()
void setWorkerRunning(boolean flag)
static synchronized IngestServices getInstance()