Autopsy  4.14.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
SolrSearchService.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2015-2019 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.keywordsearch;
20 
21 import com.google.common.eventbus.Subscribe;
22 import java.io.File;
23 import java.io.IOException;
24 import java.io.Reader;
25 import java.lang.reflect.InvocationTargetException;
26 import java.net.InetAddress;
27 import java.util.ArrayList;
28 import java.util.List;
29 import java.util.MissingResourceException;
30 import java.util.logging.Level;
31 import javax.swing.JDialog;
32 import javax.swing.JOptionPane;
33 import javax.swing.SwingUtilities;
34 import org.apache.commons.io.FileUtils;
35 import org.apache.commons.lang.math.NumberUtils;
36 import org.apache.solr.client.solrj.SolrServerException;
37 import org.apache.solr.client.solrj.impl.HttpSolrServer;
38 import org.openide.util.NbBundle;
39 import org.openide.util.lookup.ServiceProvider;
40 import org.openide.util.lookup.ServiceProviders;
53 import org.sleuthkit.datamodel.Blackboard;
54 import org.sleuthkit.datamodel.BlackboardArtifact;
55 import org.sleuthkit.datamodel.Content;
56 import org.sleuthkit.datamodel.TskCoreException;
57 
62 @ServiceProviders(value = {
63  @ServiceProvider(service = KeywordSearchService.class),
64  @ServiceProvider(service = AutopsyService.class)
65 })
67 
68  private static final String BAD_IP_ADDRESS_FORMAT = "ioexception occurred when talking to server"; //NON-NLS
69  private static final String SERVER_REFUSED_CONNECTION = "server refused connection"; //NON-NLS
70  private static final int IS_REACHABLE_TIMEOUT_MS = 1000;
71  private static final int LARGE_INDEX_SIZE_GB = 50;
72  private static final int GIANT_INDEX_SIZE_GB = 500;
73  private static final Logger logger = Logger.getLogger(SolrSearchService.class.getName());
74 
90  @Override
91  public void index(Content content) throws TskCoreException {
92  /*
93  * TODO (JIRA-1099): The following code has some issues that need to be
94  * resolved. For artifacts, it is assumed that the posting of artifacts
95  * is only occuring during an ingest job with an enabled keyword search
96  * ingest module handling index commits; it also assumes that the
97  * artifacts are only posted by modules in the either the file level
98  * ingest pipeline or the first stage data source level ingest pipeline,
99  * so that the artifacts will be searched during a periodic or final
100  * keyword search. It also assumes that the only other type of Content
101  * for which this API will be called are Reports generated at a time
102  * when doing a commit is required and desirable, i.e., in a context
103  * other than an ingest job.
104  */
105  if (content == null) {
106  return;
107  }
108  final Ingester ingester = Ingester.getDefault();
109  if (content instanceof BlackboardArtifact) {
110  BlackboardArtifact artifact = (BlackboardArtifact) content;
111  if (artifact.getArtifactID() > 0) {
112  /*
113  * Artifact indexing is only supported for artifacts that use
114  * negative artifact ids to avoid overlapping with the object
115  * ids of other types of Content.
116  */
117  return;
118  }
119  try {
120  TextExtractor blackboardExtractor = TextExtractorFactory.getExtractor(content, null);
121  Reader blackboardExtractedTextReader = blackboardExtractor.getReader();
122  String sourceName = artifact.getDisplayName() + "_" + artifact.getArtifactID();
123  ingester.indexMetaDataOnly(artifact, sourceName);
124  ingester.indexText(blackboardExtractedTextReader, artifact.getArtifactID(), sourceName, content, null);
125  } catch (Ingester.IngesterException | TextExtractorFactory.NoTextExtractorFound | TextExtractor.InitReaderException ex) {
126  throw new TskCoreException("Error indexing artifact", ex);
127  }
128  } else {
129  try {
130  TextExtractor contentExtractor = TextExtractorFactory.getExtractor(content, null);
131  Reader contentExtractedTextReader = contentExtractor.getReader();
132  ingester.indexText(contentExtractedTextReader, content.getId(), content.getName(), content, null);
133  } catch (TextExtractorFactory.NoTextExtractorFound | Ingester.IngesterException | TextExtractor.InitReaderException ex) {
134  try {
135  // Try the StringsTextExtractor if Tika extractions fails.
136  TextExtractor stringsExtractor = TextExtractorFactory.getStringsExtractor(content, null);
137  Reader stringsExtractedTextReader = stringsExtractor.getReader();
138  ingester.indexText(stringsExtractedTextReader, content.getId(), content.getName(), content, null);
139  } catch (Ingester.IngesterException | TextExtractor.InitReaderException ex1) {
140  throw new TskCoreException("Error indexing content", ex1);
141  }
142  }
143  ingester.commit();
144  }
145  }
146 
155  @Override
156  public void tryConnect(String host, int port) throws KeywordSearchServiceException {
157  HttpSolrServer solrServer = null;
158  if (host == null || host.isEmpty()) {
159  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.MissingHostname")); //NON-NLS
160  }
161  try {
162  solrServer = new HttpSolrServer("http://" + host + ":" + Integer.toString(port) + "/solr"); //NON-NLS
163  KeywordSearch.getServer().connectToSolrServer(solrServer);
164  } catch (SolrServerException ex) {
165  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort")); //NON-NLS
166  } catch (IOException ex) {
167  String result = NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort"); //NON-NLS
168  String message = ex.getCause().getMessage().toLowerCase();
169  if (message.startsWith(SERVER_REFUSED_CONNECTION)) {
170  try {
171  if (InetAddress.getByName(host).isReachable(IS_REACHABLE_TIMEOUT_MS)) {
172  // if we can reach the host, then it's probably port problem
173  result = Bundle.SolrConnectionCheck_Port();
174  } else {
175  result = NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort"); //NON-NLS
176  }
177  } catch (IOException | MissingResourceException any) {
178  // it may be anything
179  result = NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort"); //NON-NLS
180  }
181  } else if (message.startsWith(BAD_IP_ADDRESS_FORMAT)) {
182  result = NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.Hostname"); //NON-NLS
183  }
184  throw new KeywordSearchServiceException(result);
185  } catch (NumberFormatException ex) {
186  throw new KeywordSearchServiceException(Bundle.SolrConnectionCheck_Port());
187  } catch (IllegalArgumentException ex) {
188  throw new KeywordSearchServiceException(ex.getMessage());
189  } finally {
190  if (null != solrServer) {
191  solrServer.shutdown();
192  }
193  }
194  }
195 
204  @Override
205  public void deleteDataSource(Long dataSourceId) throws KeywordSearchServiceException {
206 
207  try {
208  Server ddsServer = KeywordSearch.getServer();
209  ddsServer.deleteDataSource(dataSourceId);
210  } catch (IOException | KeywordSearchModuleException | NoOpenCoreException | SolrServerException ex) {
211  logger.log(Level.WARNING, NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.DeleteDataSource.msg", dataSourceId), ex);
212  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.DeleteDataSource.msg", dataSourceId), ex);
213  }
214  }
215 
221  @NbBundle.Messages({
222  "# {0} - case directory", "SolrSearchService.exceptionMessage.noIndexMetadata=Unable to create IndexMetaData from case directory: {0}",
223  "SolrSearchService.exceptionMessage.noCurrentSolrCore=IndexMetadata did not contain a current Solr core so could not delete the case",
224  "# {0} - index folder path", "SolrSearchService.exceptionMessage.failedToDeleteIndexFiles=Failed to delete text index files at {0}"
225  })
226  @Override
228  String caseDirectory = metadata.getCaseDirectory();
229  IndexMetadata indexMetadata;
230  try {
231  indexMetadata = new IndexMetadata(caseDirectory);
232  } catch (IndexMetadata.TextIndexMetadataException ex) {
233  logger.log(Level.WARNING, NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.exceptionMessage.noIndexMetadata", caseDirectory), ex);
234  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.exceptionMessage.noIndexMetadata", caseDirectory), ex);
235  }
236  //find the index for the current version of solr (the one we are connected to) and delete its core using the index name
237  String currentSchema = IndexFinder.getCurrentSchemaVersion();
238  String currentSolr = IndexFinder.getCurrentSolrVersion();
239  for (Index index : indexMetadata.getIndexes()) {
240  if (index.getSolrVersion().equals(currentSolr) && index.getSchemaVersion().equals(currentSchema)) {
241  /*
242  * Unload/delete the core on the server and then delete the text
243  * index files.
244  */
245  KeywordSearch.getServer().deleteCore(index.getIndexName(), metadata);
246  if (!FileUtil.deleteDir(new File(index.getIndexPath()).getParentFile())) {
247  throw new KeywordSearchServiceException(Bundle.SolrSearchService_exceptionMessage_failedToDeleteIndexFiles(index.getIndexPath()));
248  }
249  }
250  return; //only one core exists for each combination of solr and schema version
251  }
252 
253  //this code this code will only execute if an index for the current core was not found
254  logger.log(Level.WARNING, NbBundle.getMessage(SolrSearchService.class,
255  "SolrSearchService.exceptionMessage.noCurrentSolrCore"));
256  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class,
257  "SolrSearchService.exceptionMessage.noCurrentSolrCore"));
258  }
259 
260  @Override
261  public String getServiceName() {
262  return NbBundle.getMessage(this.getClass(), "SolrSearchService.ServiceName");
263  }
264 
273  @Override
274  @NbBundle.Messages({
275  "SolrSearch.lookingForMetadata.msg=Looking for text index metadata file",
276  "SolrSearch.readingIndexes.msg=Reading text index metadata file",
277  "SolrSearch.findingIndexes.msg=Looking for existing text index directories",
278  "SolrSearch.creatingNewIndex.msg=Creating new text index",
279  "SolrSearch.checkingForLatestIndex.msg=Looking for text index with latest Solr and schema version",
280  "SolrSearch.indentifyingIndex.msg=Identifying text index to use",
281  "SolrSearch.openCore.msg=Opening text index",
282  "SolrSearch.openLargeCore.msg=Opening text index. This may take several minutes.",
283  "SolrSearch.openGiantCore.msg=Opening text index. Text index for this case is very large and may take long time to load.",
284  "SolrSearch.complete.msg=Text index successfully opened"})
286  if (context.cancelRequested()) {
287  return;
288  }
289 
290  ProgressIndicator progress = context.getProgressIndicator();
291  int totalNumProgressUnits = 7;
292  int progressUnitsCompleted = 0;
293 
294  String caseDirPath = context.getCase().getCaseDirectory();
295  Case theCase = context.getCase();
296  List<Index> indexes = new ArrayList<>();
297  progress.progress(Bundle.SolrSearch_lookingForMetadata_msg(), totalNumProgressUnits);
298  if (IndexMetadata.isMetadataFilePresent(caseDirPath)) {
299  try {
300  // metadata file exists, get list of existing Solr cores for this case
301  progressUnitsCompleted++;
302  progress.progress(Bundle.SolrSearch_findingIndexes_msg(), progressUnitsCompleted);
303  IndexMetadata indexMetadata = new IndexMetadata(caseDirPath);
304  indexes = indexMetadata.getIndexes();
305  } catch (IndexMetadata.TextIndexMetadataException ex) {
306  logger.log(Level.SEVERE, String.format("Unable to read text index metadata file"), ex);
307  throw new AutopsyServiceException("Unable to read text index metadata file", ex);
308  }
309  } else {
310  // metadata file doesn't exist.
311  // do case subdirectory search to look for Solr 4 Schema 1.8 indexes
312  progressUnitsCompleted++;
313  progress.progress(Bundle.SolrSearch_findingIndexes_msg(), progressUnitsCompleted);
314  Index oldIndex = IndexFinder.findOldIndexDir(theCase);
315  if (oldIndex != null) {
316  // add index to the list of indexes that exist for this case
317  indexes.add(oldIndex);
318  }
319  }
320 
321  if (context.cancelRequested()) {
322  return;
323  }
324 
325  // check if we found any existing indexes
326  Index currentVersionIndex = null;
327  if (indexes.isEmpty()) {
328  // new case that doesn't have an existing index. create new index folder
329  progressUnitsCompleted++;
330  progress.progress(Bundle.SolrSearch_creatingNewIndex_msg(), progressUnitsCompleted);
331  currentVersionIndex = IndexFinder.createLatestVersionIndexDir(theCase);
332  // add current index to the list of indexes that exist for this case
333  indexes.add(currentVersionIndex);
334  } else {
335  // check if one of the existing indexes is for latest Solr version and schema
336  progressUnitsCompleted++;
337  progress.progress(Bundle.SolrSearch_checkingForLatestIndex_msg(), progressUnitsCompleted);
338  currentVersionIndex = IndexFinder.findLatestVersionIndexDir(indexes);
339  if (currentVersionIndex == null) {
340  // found existing index(es) but none were for latest Solr version and schema version
341  progressUnitsCompleted++;
342  progress.progress(Bundle.SolrSearch_indentifyingIndex_msg(), progressUnitsCompleted);
343  Index indexToUse = IndexFinder.identifyIndexToUse(indexes);
344  if (indexToUse == null) {
345  // unable to find index that can be used
346  throw new AutopsyServiceException("Unable to find index that can be used for this case");
347  }
348 
349  if (context.cancelRequested()) {
350  return;
351  }
352 
353  double currentSolrVersion = NumberUtils.toDouble(IndexFinder.getCurrentSolrVersion());
354  double indexSolrVersion = NumberUtils.toDouble(indexToUse.getSolrVersion());
355  if (indexSolrVersion == currentSolrVersion) {
356  // latest Solr version but schema not compatible. index should be used in read-only mode
357  if (!indexToUse.isCompatible(IndexFinder.getCurrentSchemaVersion()) && RuntimeProperties.runningWithGUI()) {
358  // pop up a message box to indicate the read-only restrictions.
359  JOptionPane optionPane = new JOptionPane(
360  NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexReadOnlyDialog.msg"),
361  JOptionPane.WARNING_MESSAGE,
362  JOptionPane.DEFAULT_OPTION);
363  try {
364  SwingUtilities.invokeAndWait(() -> {
365  JDialog dialog = optionPane.createDialog(NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexReadOnlyDialog.title"));
366  dialog.setVisible(true);
367  });
368  } catch (InterruptedException ex) {
369  // Cancelled
370  return;
371  } catch (InvocationTargetException ex) {
372  throw new AutopsyServiceException("Error displaying limited search features warning dialog", ex);
373  }
374  }
375  // proceed with case open
376  currentVersionIndex = indexToUse;
377  } else {
378  // index needs to be upgraded to latest supported version of Solr
379  throw new AutopsyServiceException("Unable to find index to use for Case open");
380  }
381  }
382  }
383 
384  try {
385  // update text index metadata file
386  if (!indexes.isEmpty()) {
387  IndexMetadata indexMetadata = new IndexMetadata(caseDirPath, indexes);
388  }
389  } catch (IndexMetadata.TextIndexMetadataException ex) {
390  throw new AutopsyServiceException("Failed to save Solr core info in text index metadata file", ex);
391  }
392 
393  // open core
394  try {
395  // check text index size to gauge estimated time to open/load the index
396  long indexSizeInBytes = FileUtils.sizeOfDirectory(new File(currentVersionIndex.getIndexPath()));
397  long sizeInGb = indexSizeInBytes / 1000000000;
398  if (sizeInGb < LARGE_INDEX_SIZE_GB) {
399  progress.progress(Bundle.SolrSearch_openCore_msg(), totalNumProgressUnits - 1);
400  } else if (sizeInGb >= LARGE_INDEX_SIZE_GB && sizeInGb < GIANT_INDEX_SIZE_GB) {
401  progress.switchToIndeterminate(Bundle.SolrSearch_openLargeCore_msg());
402  } else {
403  progress.switchToIndeterminate(Bundle.SolrSearch_openGiantCore_msg());
404  }
405 
406  KeywordSearch.getServer().openCoreForCase(theCase, currentVersionIndex);
407  } catch (KeywordSearchModuleException ex) {
408  throw new AutopsyServiceException(String.format("Failed to open or create core for %s", caseDirPath), ex);
409  }
410  if (context.cancelRequested()) {
411  return;
412  }
413 
414  theCase.getSleuthkitCase().registerForEvents(this);
415 
416  progress.progress(Bundle.SolrSearch_complete_msg(), totalNumProgressUnits);
417  }
418 
427  @Override
429  /*
430  * TODO (JIRA 2525): The following code KeywordSearch.CaseChangeListener
431  * gambles that any BlackboardResultWriters (SwingWorkers) will complete
432  * in less than roughly two seconds. This stuff should be reworked using
433  * an ExecutorService and tasks with Futures.
434  */
435  AdHocSearchChildFactory.BlackboardResultWriter.stopAllWriters();
436  try {
437  Thread.sleep(2000);
438  } catch (InterruptedException ex) {
439  logger.log(Level.SEVERE, "Unexpected interrupt while waiting for BlackboardResultWriters to terminate", ex);
440  }
441 
442  try {
443  KeywordSearch.getServer().closeCore();
444  } catch (KeywordSearchModuleException ex) {
445  throw new AutopsyServiceException(String.format("Failed to close core for %s", context.getCase().getCaseDirectory()), ex);
446  }
447 
448  context.getCase().getSleuthkitCase().unregisterForEvents(this);
449  }
450 
456  @NbBundle.Messages("SolrSearchService.indexingError=Unable to index blackboard artifact.")
457  @Subscribe
458  void handleNewArtifacts(Blackboard.ArtifactsPostedEvent event) {
459  for (BlackboardArtifact artifact : event.getArtifacts()) {
460  if ((artifact.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID()) && // don't index KWH bc it's based on existing indexed text
461  (artifact.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_ASSOCIATED_OBJECT.getTypeID())){ //don't index AO bc it has only an artifact ID - no useful text
462  try {
463  index(artifact);
464  } catch (TskCoreException ex) {
465  //TODO: is this the right error handling?
466  logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS
467  MessageNotifyUtil.Notify.error(Bundle.SolrSearchService_indexingError(), artifact.getDisplayName());
468  }
469  }
470  }
471  }
472 
482  @Deprecated
483  @Override
484  public void indexArtifact(BlackboardArtifact artifact) throws TskCoreException {
485  if (artifact == null) {
486  return;
487  }
488 
489  // We only support artifact indexing for Autopsy versions that use
490  // the negative range for artifact ids.
491  if (artifact.getArtifactID() > 0) {
492  return;
493  }
494  final Ingester ingester = Ingester.getDefault();
495 
496  try {
497  String sourceName = artifact.getDisplayName() + "_" + artifact.getArtifactID();
498  TextExtractor blackboardExtractor = TextExtractorFactory.getExtractor((Content) artifact, null);
499  Reader blackboardExtractedTextReader = blackboardExtractor.getReader();
500  ingester.indexMetaDataOnly(artifact, sourceName);
501  ingester.indexText(blackboardExtractedTextReader, artifact.getId(), sourceName, artifact, null);
502  } catch (Ingester.IngesterException | TextExtractorFactory.NoTextExtractorFound | TextExtractor.InitReaderException ex) {
503  throw new TskCoreException(ex.getCause().getMessage(), ex);
504  }
505  }
506 
507 }
static TextExtractor getStringsExtractor(Content content, Lookup context)
static TextExtractor getExtractor(Content content, Lookup context)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
static boolean deleteDir(File dirPath)
Definition: FileUtil.java:47

Copyright © 2012-2020 Basis Technology. Generated on: Wed Apr 8 2020
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.